diff --git a/services/bstein-dev-home/av-observer/index.html b/services/bstein-dev-home/av-observer/index.html index 5db27f16..a6101484 100644 --- a/services/bstein-dev-home/av-observer/index.html +++ b/services/bstein-dev-home/av-observer/index.html @@ -9,6 +9,7 @@
+

Stopped

Reporting starts with the test.

diff --git a/services/bstein-dev-home/av-observer/kustomization.yaml b/services/bstein-dev-home/av-observer/kustomization.yaml index b2169498..bb6b6efd 100644 --- a/services/bstein-dev-home/av-observer/kustomization.yaml +++ b/services/bstein-dev-home/av-observer/kustomization.yaml @@ -15,6 +15,10 @@ configMapGenerator: - metrics.js - reporting.js - activity-worklet.js + - paired-observer.js + - paired-worklet.js + - paired-signals.js + - test-signals.js - style.css - name: lesavka-av-observer-receipts files: diff --git a/services/bstein-dev-home/av-observer/nginx.conf b/services/bstein-dev-home/av-observer/nginx.conf index e6b9cd3d..66de020a 100644 --- a/services/bstein-dev-home/av-observer/nginx.conf +++ b/services/bstein-dev-home/av-observer/nginx.conf @@ -34,7 +34,7 @@ http { if ($request_method !~ ^(GET|HEAD)$) { return 405; } try_files /av-test/index.html =404; } - location ~ ^/av-test/(index\.html|observer\.js|metrics\.js|reporting\.js|activity-worklet\.js|style\.css)$ { + location ~ ^/av-test/(index\.html|observer\.js|metrics\.js|reporting\.js|activity-worklet\.js|paired-observer\.js|paired-worklet\.js|paired-signals\.js|test-signals\.js|style\.css)$ { if ($request_method !~ ^(GET|HEAD)$) { return 405; } try_files $uri =404; } diff --git a/services/bstein-dev-home/av-observer/observer.js b/services/bstein-dev-home/av-observer/observer.js index 368a0004..a44f8475 100644 --- a/services/bstein-dev-home/av-observer/observer.js +++ b/services/bstein-dev-home/av-observer/observer.js @@ -1,5 +1,6 @@ import {VideoMetrics,receipt} from './metrics.js'; import {RunReporter} from './reporting.js'; +import {PairedObserver} from './paired-observer.js'; const el=id=>document.getElementById(id); let generation=0, current=null, last=null, reportingRun=null; const stopTracks=stream=>stream?.getTracks().forEach(track=>track.stop()); @@ -19,6 +20,7 @@ function stop(reason='operator') { if(run) { release(run); last=receipt(run.video,run.audio,run.settings,performance.now()-run.started,reason); + if(run.paired)last.paired=run.paired.summary(); run.reporter.finish(last); el('metrics').textContent=JSON.stringify(last,null,2); el('export').disabled=false; @@ -26,6 +28,7 @@ function stop(reason='operator') { el('preview').srcObject=null; el('start').disabled=!el('audio-device').value || !el('video-device').value; el('stop').disabled=true; el('devices').disabled=false; + el('paired-signal').disabled=false; el('status').textContent=`Stopped · ${reason}`; } el('devices').onclick=async()=>{ @@ -61,13 +64,14 @@ el('start').onclick=async()=>{ el('status').textContent='This browser lacks the required observation APIs.';return; } const ticket=++generation; - const run={started:performance.now(),video:new VideoMetrics(),audio:null,settings:{}}; + const run={started:performance.now(),video:new VideoMetrics(),audio:null,settings:{},pairedEnabled:el('paired-signal').checked}; current=run; last=null; reportingRun=run; run.reporter=new RunReporter(message=>{ if(reportingRun===run)el('report-status').textContent=message; }); el('start').disabled=true; el('devices').disabled=true; el('stop').disabled=false; el('export').disabled=true; + el('paired-signal').disabled=true; el('status').textContent='Starting…'; run.setupTimer=setTimeout(()=>{if(current===run)stop('setup timeout');},20000); try { @@ -83,14 +87,17 @@ el('start').onclick=async()=>{ audio:{sampleRate:as.sampleRate,channelCount:as.channelCount,echoCancellation:as.echoCancellation, noiseSuppression:as.noiseSuppression,autoGainControl:as.autoGainControl}}; run.context=new AudioContext(); - await run.context.audioWorklet.addModule('activity-worklet.js'); + await run.context.audioWorklet.addModule(run.pairedEnabled?'paired-worklet.js':'activity-worklet.js'); if(generation!==ticket)return; - run.node=new AudioWorkletNode(run.context,'av-activity'); + if(run.pairedEnabled)run.paired=new PairedObserver(run.context,el('preview')); + run.node=new AudioWorkletNode(run.context,run.pairedEnabled?'av-paired-signals':'av-activity'); run.node.onprocessorerror=()=>{if(current===run)stop('audio observer error');}; run.node.port.onmessage=({data})=>{ if(current!==run)return; - run.audio=data; run.node.port.postMessage('ack'); + if(run.paired){run.audio=data.activity;run.paired.audioMessage(data);}else run.audio=data; + run.node.port.postMessage('ack'); const sample=receipt(run.video,run.audio,run.settings,performance.now()-run.started,'observing'); + if(run.paired)sample.paired=run.paired.summary(); el('metrics').textContent=JSON.stringify(sample,null,2); if(run.observing)run.reporter.observe(sample); }; @@ -107,6 +114,7 @@ el('start').onclick=async()=>{ function frame(now,metadata) { if(current!==run)return; run.video.observe(metadata.expectedDisplayTime,metadata.presentedFrames); + run.paired?.frame(metadata.expectedDisplayTime); run.callback=el('preview').requestVideoFrameCallback(frame); } run.callback=el('preview').requestVideoFrameCallback(frame); diff --git a/services/bstein-dev-home/av-observer/paired-observer.js b/services/bstein-dev-home/av-observer/paired-observer.js new file mode 100644 index 00000000..c9024436 --- /dev/null +++ b/services/bstein-dev-home/av-observer/paired-observer.js @@ -0,0 +1,53 @@ +import {colorCode,PulseEdges} from './test-signals.js'; +import {pairedSummary} from './paired-signals.js'; + +// Browser adapter. Source/device timestamps are not supplied by these APIs; +// this clock map is only a processing-time estimate, never release evidence. +// https://www.w3.org/TR/webaudio/#dom-baseaudiocontext-currenttime +// https://wicg.github.io/video-rvfc/ +export class PairedObserver { + constructor(context,video) { + this.context=context;this.video=video;this.audio=[];this.frames=[]; + this.edges=new PulseEdges();this.previous=null;this.overflow=0;this.clockInvalid=false; + this.canvas=document.createElement('canvas');this.canvas.width=16;this.canvas.height=16; + this.drawing=this.canvas.getContext('2d',{willReadFrequently:true}); + this.offset=null; + } + audioMessage(data) { + if(!Array.isArray(data.signals)||data.signals.length>32){this.clockInvalid=true;return;} + this.overflow=Math.max(this.overflow,data.overflow??0); + const before=performance.now(),clock=this.context.currentTime*1000,after=performance.now(); + if(this.context.state!=='running'||after-before>5||!Number.isFinite(clock)){this.clockInvalid=true;return;} + const offset=(before+after)/2-clock; + if(this.offset!==null&&Math.abs(offset-this.offset)>25)this.clockInvalid=true; + this.offset=offset; + for(const event of data.signals) { + if(this.audio.length>=256){this.overflow++;continue;} + this.audio.push({code:event.code,startMs:event.startMs+offset,endMs:event.endMs+offset}); + } + } + frame(timeMs) { + if(!this.drawing||!this.video.videoWidth||!this.video.videoHeight)return; + let pixels; + try { + // Small transient centre sample. Clear both the canvas and pixel + // array immediately: no retained image, recording or upload. + const w=this.video.videoWidth,h=this.video.videoHeight; + this.drawing.drawImage(this.video,w/4,h/4,w/2,h/2,0,0,16,16); + pixels=this.drawing.getImageData(0,0,16,16).data; + const code=colorCode(pixels); + const previous=this.previous;this.previous=timeMs; + if(previous===null||timeMs<=previous||timeMs-previous>250)return; + const event=this.edges.observe(code,previous,timeMs); + if(event){if(this.frames.length<256)this.frames.push(event);else this.overflow++;} + } catch {this.clockInvalid=true;} + finally {pixels?.fill(0);this.drawing.clearRect(0,0,16,16);} + } + summary() { + const result=pairedSummary(this.audio,this.frames); + if(this.clockInvalid||this.overflow)result.estimatedMedianSignedMs=null; + return {...result,audioSignals:this.audio.length,videoSignals:this.frames.length, + overflow:this.overflow,clockInvalid:this.clockInvalid, + note:'Coded signal / browser processing-time estimate only; input-clock calibration and source age are unmeasured.'}; + } +} diff --git a/services/bstein-dev-home/av-observer/paired-signals.js b/services/bstein-dev-home/av-observer/paired-signals.js new file mode 100644 index 00000000..d62796f3 --- /dev/null +++ b/services/bstein-dev-home/av-observer/paired-signals.js @@ -0,0 +1,45 @@ +// Pair already-recognized synthetic identities in a COMMON measured clock. +// Refuse multiple same-code candidates instead of choosing the smallest skew. +// Repeated palettes without an unambiguous time bound are not unique IDs. +const valid=e=>e&&Number.isInteger(e.code)&&e.code>=1&&e.code<=16&& + Number.isFinite(e.startMs)&&Number.isFinite(e.endMs)&&e.startMs>=0&&e.endMs>=e.startMs&& + e.endMs-e.startMs<=250; +const quantile=(values,p)=>values.length?[...values].sort((a,b)=>a-b)[Math.ceil(values.length*p)-1]:null; + +export function pairedSummary(audio,video,{clockUncertaintyMs=null,maxSkewMs=2000}={}) { + const result={pairs:0,ambiguous:0,unpairedAudio:audio.length,unpairedVideo:video.length, + skewAbsP95UpperMs:null,skewAbsMaxUpperMs:null,medianSignedMs:null,estimatedMedianSignedMs:null, + clockUncertaintyMs:null,releaseEligible:false}; + if(audio.length>256||video.length>256||!audio.every(valid)||!video.every(valid)|| + !Number.isFinite(maxSkewMs)||maxSkewMs<=0||maxSkewMs>10000)return result; + for(const events of [audio,video]) { + if(events.some((e,i)=>i&&e.startMs<=events[i-1].startMs))return result; + } + const candidates=audio.map(a=>video.map((v,i)=>a.code===v.code&& + Math.max(a.startMs-v.endMs,v.startMs-a.endMs)<=maxSkewMs?i:-1).filter(i=>i>=0)); + const reverse=video.map((_,i)=>candidates.reduce((n,c)=>n+Number(c.includes(i)),0)); + const pairs=[];let lastVideo=-1; + for(let i=0;i(a.startMs+a.endMs-v.startMs-v.endMs)/2),.5); + // Identity matches without independently bounded clock alignment must not + // be converted into a calibrated skew or accepted qualification result. + if(typeof clockUncertaintyMs!=='number'||!Number.isFinite(clockUncertaintyMs)||clockUncertaintyMs<0||clockUncertaintyMs>1000)return result; + result.clockUncertaintyMs=clockUncertaintyMs; + const signed=[],bounds=[]; + for(const [a,v] of pairs) { + // Positive = audio later. Interval arithmetic includes detection error. + signed.push((a.startMs+a.endMs-v.startMs-v.endMs)/2); + bounds.push(Math.max(Math.abs(a.startMs-v.endMs),Math.abs(a.endMs-v.startMs))+clockUncertaintyMs); + } + result.medianSignedMs=quantile(signed,.5); + result.skewAbsP95UpperMs=quantile(bounds,.95); + result.skewAbsMaxUpperMs=bounds.length?Math.max(...bounds):null; + return result; +} diff --git a/services/bstein-dev-home/av-observer/paired-worklet.js b/services/bstein-dev-home/av-observer/paired-worklet.js new file mode 100644 index 00000000..0b7e5b0a --- /dev/null +++ b/services/bstein-dev-home/av-observer/paired-worklet.js @@ -0,0 +1,31 @@ +import {ToneDetector} from './test-signals.js'; + +// Explicit paired-signal mode only. Bounded numeric observations; never PCM. +class PairedSignals extends AudioWorkletProcessor { + constructor() { + super();this.detector=new ToneDetector(sampleRate); + this.frames=0;this.inputFrames=0;this.zeros=0;this.energy=0;this.peak=0; + this.events=[];this.overflow=0;this.credit=true;this.next=0;this.stopped=false; + this.port.onmessage=({data})=>{if(data==='ack')this.credit=true;if(data==='stop')this.stopped=true;}; + } + process(inputs,outputs) { + if(this.stopped||this.frames>=sampleRate*125)return false; + this.frames+=outputs[0]?.[0]?.length??128; + const samples=inputs[0]?.[0]; + if(samples) { + for(const x of samples){this.inputFrames++;this.energy+=x*x;this.peak=Math.max(this.peak,Math.abs(x));if(x===0)this.zeros++;} + for(const event of this.detector.process(samples,currentFrame)) { + if(this.events.length<32)this.events.push(event);else this.overflow++; + } + } + if(this.credit&&this.frames>=this.next) { + this.port.postMessage({activity:{inputFrames:this.inputFrames,processedFrames:this.frames, + exactZeroFrames:this.zeros,rms:this.inputFrames?Math.sqrt(this.energy/this.inputFrames):0, + peak:this.peak,sampleRate},signals:this.events,overflow:this.overflow}); + this.events=[];this.credit=false;this.next=this.frames+sampleRate; + } + // Browser initializes every output block to zero. No mic monitoring. + return true; + } +} +registerProcessor('av-paired-signals',PairedSignals); diff --git a/services/bstein-dev-home/av-observer/report-service/validation.py b/services/bstein-dev-home/av-observer/report-service/validation.py index 6db6d89e..99648560 100644 --- a/services/bstein-dev-home/av-observer/report-service/validation.py +++ b/services/bstein-dev-home/av-observer/report-service/validation.py @@ -11,6 +11,9 @@ BOUNDS = { "video_interval_p95_ms": 180_000, "video_gap_max_ms": 180_000, "audio_input_frames": 100_000_000, "audio_processed_frames": 100_000_000, "audio_zero_frames": 100_000_000, "audio_rms": 32, "audio_peak": 32, + "paired_audio_signals": 256, "paired_video_signals": 256, + "paired_matches": 256, "paired_ambiguous": 256, "paired_overflow": 100_000, + "paired_estimate_audio_late_ms": 10_000, "paired_estimate_audio_early_ms": 10_000, } diff --git a/services/bstein-dev-home/av-observer/reporting.js b/services/bstein-dev-home/av-observer/reporting.js index 59342a1a..7c248ab8 100644 --- a/services/bstein-dev-home/av-observer/reporting.js +++ b/services/bstein-dev-home/av-observer/reporting.js @@ -2,7 +2,8 @@ // pass a receipt, device object, error string, frame, PCM or DOM value to fetch. export function reportMetrics(receipt) { const finite=value=>typeof value==='number'&&Number.isFinite(value)&&value>=0 ? value : null; - const v=receipt.video??{},a=receipt.audio??{},s=receipt.settings??{}; + const v=receipt.video??{},a=receipt.audio??{},s=receipt.settings??{},p=receipt.paired??{}; + const estimate=typeof p.estimatedMedianSignedMs==='number'&&Number.isFinite(p.estimatedMedianSignedMs)?p.estimatedMedianSignedMs:null; return Object.fromEntries(Object.entries({ elapsed_ms:receipt.elapsedMs, video_width:s.video?.width,video_height:s.video?.height,video_fps:s.video?.frameRate, @@ -12,6 +13,10 @@ export function reportMetrics(receipt) { video_gap_max_ms:v.callbackGapMaxMs,audio_input_frames:a.inputFrames, audio_processed_frames:a.processedFrames,audio_zero_frames:a.exactZeroFrames, audio_rms:a.rms,audio_peak:a.peak,audio_context_rate:a.sampleRate, + paired_audio_signals:p.audioSignals,paired_video_signals:p.videoSignals, + paired_matches:p.pairs,paired_ambiguous:p.ambiguous,paired_overflow:p.overflow, + paired_estimate_audio_late_ms:estimate===null?null:Math.max(0,estimate), + paired_estimate_audio_early_ms:estimate===null?null:Math.max(0,-estimate), }).map(([key,value])=>[key,finite(value)])); } diff --git a/services/bstein-dev-home/av-observer/test-signals.js b/services/bstein-dev-home/av-observer/test-signals.js new file mode 100644 index 00000000..1bb9ccab --- /dev/null +++ b/services/bstein-dev-home/av-observer/test-signals.js @@ -0,0 +1,85 @@ +// Streaming recognition of the existing synthetic coded-pulse protocol. +// No samples/images are retained. A match identifies a diagnostic signal, +// not arbitrary content, a transport timestamp or a calibrated browser clock. +const tones=[620,780,940,1120,1320,1540,1780,2040,2320,2620,2960,3340,3760,4220,4740,5320]; +const colors=[[255,45,45],[0,230,118],[41,121,255],[255,179,0], + [216,27,96],[0,188,212],[205,220,57],[126,87,194], + [255,112,67],[38,166,154],[255,64,129],[92,107,192], + [255,235,59],[105,240,174],[171,71,188],[3,169,244]]; +const normalized=colors.map(rgb=>rgb.map(v=>v/Math.max(...rgb))); + +export function colorCode(rgba) { + if(!rgba || rgba.length<16 || rgba.length>4096 || rgba.length%4)return 0; + const mean=[0,0,0];const low=[255,255,255],high=[0,0,0]; + for(let i=0;i255)return 0; + mean[c]+=v;low[c]=Math.min(low[c],v);high[c]=Math.max(high[c],v); + } + for(let c=0;c<3;c++)mean[c]/=rgba.length/4; + const max=Math.max(...mean),min=Math.min(...mean); + if(max<70 || max-min<50 || high.some((v,i)=>v-low[i]>45))return 0; + const rgb=mean.map(v=>v/max); + const distances=normalized.map((p,index)=>({code:index+1, + distance:Math.sqrt(p.reduce((sum,v,i)=>sum+(rgb[i]-v)**2,0))})).sort((a,b)=>a.distance-b.distance); + if(distances[0].distance>.13 || distances[1].distance-distances[0].distance<.035)return 0; + return distances[0].code; +} + +// Two consecutive observations confirm an onset; short detector dropouts do +// not manufacture another event. start/end bound the FIRST matching window. +export class PulseEdges { + constructor(){this.active=0;this.candidate=null;this.off=0;this.last=-Infinity;this.lastStart=-Infinity;} + observe(code,startMs,endMs) { + if(!Number.isInteger(code)||code<0||code>16||!Number.isFinite(startMs)|| + !Number.isFinite(endMs)||startMs<0||endMs=2){this.active=0;this.candidate=null;}return null;} + this.off=0; + if(code===this.active)return null; + if(this.candidate?.code!==code){this.candidate={code,startMs,endMs};return null;} + const result=this.candidate;this.active=code;this.candidate=null;return result; + } +} + +export class ToneDetector { + constructor(rate) { + if(!Number.isFinite(rate)||rate<16000||rate>192000)throw new RangeError('sample rate'); + this.rate=rate;this.size=Math.round(rate*.01);this.coeff=tones.map(f=>2*Math.cos(2*Math.PI*f/rate)); + this.q1=new Float64Array(16);this.q2=new Float64Array(16); + this.edges=new PulseEdges();this.expected=null;this.reset(); + } + reset(){this.q1.fill(0);this.q2.fill(0);this.count=0;this.energy=0;} + // frame is AudioWorklet currentFrame, not message delivery time. + process(samples,frame) { + if(!Number.isSafeInteger(frame)||frame<0||samples.length>8192)throw new RangeError('audio block'); + if(this.expected!==null&&frame!==this.expected){this.reset();this.edges=new PulseEdges();} + this.expected=frame+samples.length; + const events=[]; + for(let n=0;nbest){second=best;best=power;code=i+1;}else second=Math.max(second,power); + } + if(this.energy/this.size<1e-6 || 2*best/(this.size*this.energy)<.75 || best