titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

av-test: add bounded numeric paired-signal diagnostics

Browse Source
This commit is contained in:
jenkins 2026-10-03 16:21:28 -05:00
parent 0ad4ec866b
commit 5d50d2f8c8
10 changed files with 241 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
services/bstein-dev-home/av-observer/index.html
View File

@ -9,6 +9,7 @@
<label>Video <select id="video-device"></select></label>
<label>Microphone <select id="audio-device"></select></label></div>
<div><button id="start" disabled>Start 2-minute check</button><button id="stop" disabled>Stop</button><button id="export" disabled>Save metrics</button></div>
<label title="Recognize externally supplied coded colors and tones. Reports a diagnostic processing-time estimate, not calibrated lip sync or latency."><input id="paired-signal" type="checkbox"> Paired test signal</label>
<p id="status" role="status">Stopped</p>
<p id="report-status" role="status">Reporting starts with the test.</p>
<video id="preview" autoplay muted playsinline></video>

4
services/bstein-dev-home/av-observer/kustomization.yaml
View File

@ -15,6 +15,10 @@ configMapGenerator:
- metrics.js
- reporting.js
- activity-worklet.js
- paired-observer.js
- paired-worklet.js
- paired-signals.js
- test-signals.js
- style.css
- name: lesavka-av-observer-receipts
files:

2
services/bstein-dev-home/av-observer/nginx.conf
View File

@ -34,7 +34,7 @@ http {
if ($request_method !~ ^(GET|HEAD)$) { return 405; }
try_files /av-test/index.html =404;
}
location ~ ^/av-test/(index\.html|observer\.js|metrics\.js|reporting\.js|activity-worklet\.js|style\.css)$ {
location ~ ^/av-test/(index\.html|observer\.js|metrics\.js|reporting\.js|activity-worklet\.js|paired-observer\.js|paired-worklet\.js|paired-signals\.js|test-signals\.js|style\.css)$ {
if ($request_method !~ ^(GET|HEAD)$) { return 405; }
try_files $uri =404;
}

16
services/bstein-dev-home/av-observer/observer.js
View File

@ -1,5 +1,6 @@
import {VideoMetrics,receipt} from './metrics.js';
import {RunReporter} from './reporting.js';
import {PairedObserver} from './paired-observer.js';
const el=id=>document.getElementById(id);
let generation=0, current=null, last=null, reportingRun=null;
const stopTracks=stream=>stream?.getTracks().forEach(track=>track.stop());
@ -19,6 +20,7 @@ function stop(reason='operator') {
if(run) {
release(run);
last=receipt(run.video,run.audio,run.settings,performance.now()-run.started,reason);
if(run.paired)last.paired=run.paired.summary();
run.reporter.finish(last);
el('metrics').textContent=JSON.stringify(last,null,2);
el('export').disabled=false;
@ -26,6 +28,7 @@ function stop(reason='operator') {
el('preview').srcObject=null;
el('start').disabled=!el('audio-device').value || !el('video-device').value;
el('stop').disabled=true; el('devices').disabled=false;
el('paired-signal').disabled=false;
el('status').textContent=`Stopped · ${reason}`;
}
el('devices').onclick=async()=>{
@ -61,13 +64,14 @@ el('start').onclick=async()=>{
el('status').textContent='This browser lacks the required observation APIs.';return;
}
const ticket=++generation;
const run={started:performance.now(),video:new VideoMetrics(),audio:null,settings:{}};
const run={started:performance.now(),video:new VideoMetrics(),audio:null,settings:{},pairedEnabled:el('paired-signal').checked};
current=run; last=null;
reportingRun=run;
run.reporter=new RunReporter(message=>{
if(reportingRun===run)el('report-status').textContent=message;
});
el('start').disabled=true; el('devices').disabled=true; el('stop').disabled=false; el('export').disabled=true;
el('paired-signal').disabled=true;
el('status').textContent='Starting…';
run.setupTimer=setTimeout(()=>{if(current===run)stop('setup timeout');},20000);
try {
@ -83,14 +87,17 @@ el('start').onclick=async()=>{
audio:{sampleRate:as.sampleRate,channelCount:as.channelCount,echoCancellation:as.echoCancellation,
noiseSuppression:as.noiseSuppression,autoGainControl:as.autoGainControl}};
run.context=new AudioContext();
await run.context.audioWorklet.addModule('activity-worklet.js');
await run.context.audioWorklet.addModule(run.pairedEnabled?'paired-worklet.js':'activity-worklet.js');
if(generation!==ticket)return;
run.node=new AudioWorkletNode(run.context,'av-activity');
if(run.pairedEnabled)run.paired=new PairedObserver(run.context,el('preview'));
run.node=new AudioWorkletNode(run.context,run.pairedEnabled?'av-paired-signals':'av-activity');
run.node.onprocessorerror=()=>{if(current===run)stop('audio observer error');};
run.node.port.onmessage=({data})=>{
if(current!==run)return;
run.audio=data; run.node.port.postMessage('ack');
if(run.paired){run.audio=data.activity;run.paired.audioMessage(data);}else run.audio=data;
run.node.port.postMessage('ack');
const sample=receipt(run.video,run.audio,run.settings,performance.now()-run.started,'observing');
if(run.paired)sample.paired=run.paired.summary();
el('metrics').textContent=JSON.stringify(sample,null,2);
if(run.observing)run.reporter.observe(sample);
};
@ -107,6 +114,7 @@ el('start').onclick=async()=>{
function frame(now,metadata) {
if(current!==run)return;
run.video.observe(metadata.expectedDisplayTime,metadata.presentedFrames);
run.paired?.frame(metadata.expectedDisplayTime);
run.callback=el('preview').requestVideoFrameCallback(frame);
}
run.callback=el('preview').requestVideoFrameCallback(frame);

53
services/bstein-dev-home/av-observer/paired-observer.js Normal file
View File

@ -0,0 +1,53 @@
import {colorCode,PulseEdges} from './test-signals.js';
import {pairedSummary} from './paired-signals.js';
// Browser adapter. Source/device timestamps are not supplied by these APIs;
// this clock map is only a processing-time estimate, never release evidence.
// https://www.w3.org/TR/webaudio/#dom-baseaudiocontext-currenttime
// https://wicg.github.io/video-rvfc/
export class PairedObserver {
constructor(context,video) {
this.context=context;this.video=video;this.audio=[];this.frames=[];
this.edges=new PulseEdges();this.previous=null;this.overflow=0;this.clockInvalid=false;
this.canvas=document.createElement('canvas');this.canvas.width=16;this.canvas.height=16;
this.drawing=this.canvas.getContext('2d',{willReadFrequently:true});
this.offset=null;
}
audioMessage(data) {
if(!Array.isArray(data.signals)||data.signals.length>32){this.clockInvalid=true;return;}
this.overflow=Math.max(this.overflow,data.overflow??0);
const before=performance.now(),clock=this.context.currentTime*1000,after=performance.now();
if(this.context.state!=='running'||after-before>5||!Number.isFinite(clock)){this.clockInvalid=true;return;}
const offset=(before+after)/2-clock;
if(this.offset!==null&&Math.abs(offset-this.offset)>25)this.clockInvalid=true;
this.offset=offset;
for(const event of data.signals) {
if(this.audio.length>=256){this.overflow++;continue;}
this.audio.push({code:event.code,startMs:event.startMs+offset,endMs:event.endMs+offset});
}
}
frame(timeMs) {
if(!this.drawing||!this.video.videoWidth||!this.video.videoHeight)return;
let pixels;
try {
// Small transient centre sample. Clear both the canvas and pixel
// array immediately: no retained image, recording or upload.
const w=this.video.videoWidth,h=this.video.videoHeight;
this.drawing.drawImage(this.video,w/4,h/4,w/2,h/2,0,0,16,16);
pixels=this.drawing.getImageData(0,0,16,16).data;
const code=colorCode(pixels);
const previous=this.previous;this.previous=timeMs;
if(previous===null||timeMs<=previous||timeMs-previous>250)return;
const event=this.edges.observe(code,previous,timeMs);
if(event){if(this.frames.length<256)this.frames.push(event);else this.overflow++;}
} catch {this.clockInvalid=true;}
finally {pixels?.fill(0);this.drawing.clearRect(0,0,16,16);}
}
summary() {
const result=pairedSummary(this.audio,this.frames);
if(this.clockInvalid||this.overflow)result.estimatedMedianSignedMs=null;
return {...result,audioSignals:this.audio.length,videoSignals:this.frames.length,
overflow:this.overflow,clockInvalid:this.clockInvalid,
note:'Coded signal / browser processing-time estimate only; input-clock calibration and source age are unmeasured.'};
}
}

45
services/bstein-dev-home/av-observer/paired-signals.js Normal file
View File

@ -0,0 +1,45 @@
// Pair already-recognized synthetic identities in a COMMON measured clock.
// Refuse multiple same-code candidates instead of choosing the smallest skew.
// Repeated palettes without an unambiguous time bound are not unique IDs.
const valid=e=>e&&Number.isInteger(e.code)&&e.code>=1&&e.code<=16&&
Number.isFinite(e.startMs)&&Number.isFinite(e.endMs)&&e.startMs>=0&&e.endMs>=e.startMs&&
e.endMs-e.startMs<=250;
const quantile=(values,p)=>values.length?[...values].sort((a,b)=>a-b)[Math.ceil(values.length*p)-1]:null;
export function pairedSummary(audio,video,{clockUncertaintyMs=null,maxSkewMs=2000}={}) {
const result={pairs:0,ambiguous:0,unpairedAudio:audio.length,unpairedVideo:video.length,
skewAbsP95UpperMs:null,skewAbsMaxUpperMs:null,medianSignedMs:null,estimatedMedianSignedMs:null,
clockUncertaintyMs:null,releaseEligible:false};
if(audio.length>256||video.length>256||!audio.every(valid)||!video.every(valid)||
!Number.isFinite(maxSkewMs)||maxSkewMs<=0||maxSkewMs>10000)return result;
for(const events of [audio,video]) {
if(events.some((e,i)=>i&&e.startMs<=events[i-1].startMs))return result;
}
const candidates=audio.map(a=>video.map((v,i)=>a.code===v.code&&
Math.max(a.startMs-v.endMs,v.startMs-a.endMs)<=maxSkewMs?i:-1).filter(i=>i>=0));
const reverse=video.map((_,i)=>candidates.reduce((n,c)=>n+Number(c.includes(i)),0));
const pairs=[];let lastVideo=-1;
for(let i=0;i<audio.length;i++) {
const match=candidates[i];
if(match.length!==1||reverse[match[0]]!==1){if(match.length)result.ambiguous++;continue;}
const index=match[0];
if(index<=lastVideo){result.ambiguous++;continue;}
lastVideo=index;pairs.push([audio[i],video[index]]);
}
result.pairs=pairs.length;result.unpairedAudio-=pairs.length;result.unpairedVideo-=pairs.length;
result.estimatedMedianSignedMs=quantile(pairs.map(([a,v])=>(a.startMs+a.endMs-v.startMs-v.endMs)/2),.5);
// Identity matches without independently bounded clock alignment must not
// be converted into a calibrated skew or accepted qualification result.
if(typeof clockUncertaintyMs!=='number'||!Number.isFinite(clockUncertaintyMs)||clockUncertaintyMs<0||clockUncertaintyMs>1000)return result;
result.clockUncertaintyMs=clockUncertaintyMs;
const signed=[],bounds=[];
for(const [a,v] of pairs) {
// Positive = audio later. Interval arithmetic includes detection error.
signed.push((a.startMs+a.endMs-v.startMs-v.endMs)/2);
bounds.push(Math.max(Math.abs(a.startMs-v.endMs),Math.abs(a.endMs-v.startMs))+clockUncertaintyMs);
}
result.medianSignedMs=quantile(signed,.5);
result.skewAbsP95UpperMs=quantile(bounds,.95);
result.skewAbsMaxUpperMs=bounds.length?Math.max(...bounds):null;
return result;
}

31
services/bstein-dev-home/av-observer/paired-worklet.js Normal file
View File

@ -0,0 +1,31 @@
import {ToneDetector} from './test-signals.js';
// Explicit paired-signal mode only. Bounded numeric observations; never PCM.
class PairedSignals extends AudioWorkletProcessor {
constructor() {
super();this.detector=new ToneDetector(sampleRate);
this.frames=0;this.inputFrames=0;this.zeros=0;this.energy=0;this.peak=0;
this.events=[];this.overflow=0;this.credit=true;this.next=0;this.stopped=false;
this.port.onmessage=({data})=>{if(data==='ack')this.credit=true;if(data==='stop')this.stopped=true;};
}
process(inputs,outputs) {
if(this.stopped||this.frames>=sampleRate*125)return false;
this.frames+=outputs[0]?.[0]?.length??128;
const samples=inputs[0]?.[0];
if(samples) {
for(const x of samples){this.inputFrames++;this.energy+=x*x;this.peak=Math.max(this.peak,Math.abs(x));if(x===0)this.zeros++;}
for(const event of this.detector.process(samples,currentFrame)) {
if(this.events.length<32)this.events.push(event);else this.overflow++;
}
}
if(this.credit&&this.frames>=this.next) {
this.port.postMessage({activity:{inputFrames:this.inputFrames,processedFrames:this.frames,
exactZeroFrames:this.zeros,rms:this.inputFrames?Math.sqrt(this.energy/this.inputFrames):0,
peak:this.peak,sampleRate},signals:this.events,overflow:this.overflow});
this.events=[];this.credit=false;this.next=this.frames+sampleRate;
}
// Browser initializes every output block to zero. No mic monitoring.
return true;
}
}
registerProcessor('av-paired-signals',PairedSignals);

3
services/bstein-dev-home/av-observer/report-service/validation.py
View File

@ -11,6 +11,9 @@ BOUNDS = {
"video_interval_p95_ms": 180_000, "video_gap_max_ms": 180_000,
"audio_input_frames": 100_000_000, "audio_processed_frames": 100_000_000,
"audio_zero_frames": 100_000_000, "audio_rms": 32, "audio_peak": 32,
"paired_audio_signals": 256, "paired_video_signals": 256,
"paired_matches": 256, "paired_ambiguous": 256, "paired_overflow": 100_000,
"paired_estimate_audio_late_ms": 10_000, "paired_estimate_audio_early_ms": 10_000,
}

7
services/bstein-dev-home/av-observer/reporting.js
View File

@ -2,7 +2,8 @@
// pass a receipt, device object, error string, frame, PCM or DOM value to fetch.
export function reportMetrics(receipt) {
const finite=value=>typeof value==='number'&&Number.isFinite(value)&&value>=0 ? value : null;
const v=receipt.video??{},a=receipt.audio??{},s=receipt.settings??{};
const v=receipt.video??{},a=receipt.audio??{},s=receipt.settings??{},p=receipt.paired??{};
const estimate=typeof p.estimatedMedianSignedMs==='number'&&Number.isFinite(p.estimatedMedianSignedMs)?p.estimatedMedianSignedMs:null;
return Object.fromEntries(Object.entries({
elapsed_ms:receipt.elapsedMs,
video_width:s.video?.width,video_height:s.video?.height,video_fps:s.video?.frameRate,
@ -12,6 +13,10 @@ export function reportMetrics(receipt) {
video_gap_max_ms:v.callbackGapMaxMs,audio_input_frames:a.inputFrames,
audio_processed_frames:a.processedFrames,audio_zero_frames:a.exactZeroFrames,
audio_rms:a.rms,audio_peak:a.peak,audio_context_rate:a.sampleRate,
paired_audio_signals:p.audioSignals,paired_video_signals:p.videoSignals,
paired_matches:p.pairs,paired_ambiguous:p.ambiguous,paired_overflow:p.overflow,
paired_estimate_audio_late_ms:estimate===null?null:Math.max(0,estimate),
paired_estimate_audio_early_ms:estimate===null?null:Math.max(0,-estimate),
}).map(([key,value])=>[key,finite(value)]));
}

85
services/bstein-dev-home/av-observer/test-signals.js Normal file
View File

@ -0,0 +1,85 @@
// Streaming recognition of the existing synthetic coded-pulse protocol.
// No samples/images are retained. A match identifies a diagnostic signal,
// not arbitrary content, a transport timestamp or a calibrated browser clock.
const tones=[620,780,940,1120,1320,1540,1780,2040,2320,2620,2960,3340,3760,4220,4740,5320];
const colors=[[255,45,45],[0,230,118],[41,121,255],[255,179,0],
[216,27,96],[0,188,212],[205,220,57],[126,87,194],
[255,112,67],[38,166,154],[255,64,129],[92,107,192],
[255,235,59],[105,240,174],[171,71,188],[3,169,244]];
const normalized=colors.map(rgb=>rgb.map(v=>v/Math.max(...rgb)));
export function colorCode(rgba) {
if(!rgba || rgba.length<16 || rgba.length>4096 || rgba.length%4)return 0;
const mean=[0,0,0];const low=[255,255,255],high=[0,0,0];
for(let i=0;i<rgba.length;i+=4)for(let c=0;c<3;c++) {
const v=rgba[i+c];if(!Number.isFinite(v)||v<0||v>255)return 0;
mean[c]+=v;low[c]=Math.min(low[c],v);high[c]=Math.max(high[c],v);
}
for(let c=0;c<3;c++)mean[c]/=rgba.length/4;
const max=Math.max(...mean),min=Math.min(...mean);
if(max<70 || max-min<50 || high.some((v,i)=>v-low[i]>45))return 0;
const rgb=mean.map(v=>v/max);
const distances=normalized.map((p,index)=>({code:index+1,
distance:Math.sqrt(p.reduce((sum,v,i)=>sum+(rgb[i]-v)**2,0))})).sort((a,b)=>a.distance-b.distance);
if(distances[0].distance>.13 || distances[1].distance-distances[0].distance<.035)return 0;
return distances[0].code;
}
// Two consecutive observations confirm an onset; short detector dropouts do
// not manufacture another event. start/end bound the FIRST matching window.
export class PulseEdges {
constructor(){this.active=0;this.candidate=null;this.off=0;this.last=-Infinity;this.lastStart=-Infinity;}
observe(code,startMs,endMs) {
if(!Number.isInteger(code)||code<0||code>16||!Number.isFinite(startMs)||
!Number.isFinite(endMs)||startMs<0||endMs<startMs||startMs<this.lastStart||endMs<this.last) {
this.active=0;this.candidate=null;this.off=0;return null;
}
this.last=endMs;this.lastStart=startMs;
if(!code){if(++this.off>=2){this.active=0;this.candidate=null;}return null;}
this.off=0;
if(code===this.active)return null;
if(this.candidate?.code!==code){this.candidate={code,startMs,endMs};return null;}
const result=this.candidate;this.active=code;this.candidate=null;return result;
}
}
export class ToneDetector {
constructor(rate) {
if(!Number.isFinite(rate)||rate<16000||rate>192000)throw new RangeError('sample rate');
this.rate=rate;this.size=Math.round(rate*.01);this.coeff=tones.map(f=>2*Math.cos(2*Math.PI*f/rate));
this.q1=new Float64Array(16);this.q2=new Float64Array(16);
this.edges=new PulseEdges();this.expected=null;this.reset();
}
reset(){this.q1.fill(0);this.q2.fill(0);this.count=0;this.energy=0;}
// frame is AudioWorklet currentFrame, not message delivery time.
process(samples,frame) {
if(!Number.isSafeInteger(frame)||frame<0||samples.length>8192)throw new RangeError('audio block');
if(this.expected!==null&&frame!==this.expected){this.reset();this.edges=new PulseEdges();}
this.expected=frame+samples.length;
const events=[];
for(let n=0;n<samples.length;n++) {
const x=samples[n];
if(!Number.isFinite(x)){this.reset();this.edges=new PulseEdges();continue;}
if(!this.count)this.start=frame+n;
this.energy+=x*x;
for(let i=0;i<16;i++) {
const q=x+this.coeff[i]*this.q1[i]-this.q2[i];
this.q2[i]=this.q1[i];this.q1[i]=q;
}
if(++this.count!==this.size)continue;
let best=0,second=0,code=0;
for(let i=0;i<16;i++) {
const power=this.q1[i]**2+this.q2[i]**2-this.coeff[i]*this.q1[i]*this.q2[i];
if(power>best){second=best;best=power;code=i+1;}else second=Math.max(second,power);
}
if(this.energy/this.size<1e-6 || 2*best/(this.size*this.energy)<.75 || best<second*4)code=0;
// A partial preceding window may be too weak to classify. Include
// it in the onset bound rather than pretending detection is exact.
const event=this.edges.observe(code,Math.max(0,this.start-this.size)/this.rate*1000,
(frame+n+1)/this.rate*1000);
if(event)events.push(event);
this.reset();
}
return events;
}
}