atlas-iac/testing/probes/hermes_voice_response_probe.js

// Deterministic assistant-response EXTRACTION probe for
// dockerfiles/hermes-webui-atlas-voice.js.
//
// Round 3's caption/TTS extraction was validated only against a single synthetic
// {dataset:{rawText}} node, so it never exercised the REAL rendered assistant
// turn and shipped a turn.textContent fallback that scraped the avatar "H", the
// "Hermes" author name and the "Processed 13s" worklog chip into the
// conversation caption and the one-ahead TTS synthesizer ("HHermesProcessed
// 13s"), and truncated multi-sentence replies to their first sentence.
//
// This probe builds a FAITHFUL rendered turn — matching the live build-24 DOM
// (ui.js _createAssistantTurn / renderMessages, messages.js ensureAssistantRow):
//
//   <div class="msg-row assistant-turn" data-role="assistant">
//     <div class="msg-role assistant">
//       <div class="role-icon assistant">H</div>
//       <span class="msg-role-name">Hermes</span>
//     </div>
//     <div class="assistant-turn-blocks">
//       <div class="tool-worklog-group">…Processed 13s…</div>
//       <div class="assistant-segment assistant-segment-worklog-source"
//            hidden aria-hidden="true" data-raw-text="…interim…">…</div>
//       <div class="assistant-segment" data-raw-text="ANSWER">
//         <div class="thinking-card"><div class="msg-body">REASONING</div></div>
//         <div class="msg-body">ANSWER</div>
//       </div>
//     </div>
//   </div>
//
// with a real querySelectorAll / closest / matches implementation, then drives
// the actual exported extraction + one-ahead chunker and asserts:
//   (i)   caption/TTS text is the answer BODY only — never "H"/"Hermes"/"Processed"
//   (ii)  every sentence of a multi-sentence reply reaches the chunk queue
//   (iii) a pre-settle worklog-only frame extracts to '' (so the final pump
//         retries rather than finalizing garbage into "Listening")
//
//     node hermes_voice_response_probe.js <path-to-atlas-voice.js>
'use strict';

const fs = require('fs');
const vm = require('vm');

const SCRIPT_PATH = process.argv[2];
if (!SCRIPT_PATH) {
  throw new Error('usage: hermes_voice_response_probe.js <atlas-voice.js>');
}
const SOURCE = fs.readFileSync(SCRIPT_PATH, 'utf8');

// ── Faithful minimal DOM ────────────────────────────────────────────────────
// Supports exactly what the extraction touches: className/classList, dataset
// (backed by data-* attributes so `[data-raw-text]` selects), hidden, recursive
// textContent, appendChild/children/parentNode, setAttribute/getAttribute, and
// querySelector(All)/closest/matches over comma-separated compound selectors of
// tag / .class / [attr] / [attr="value"] terms (no combinators — none are used).

function parseSelector(selector) {
  return String(selector).split(',').map((group) => {
    const term = group.trim();
    const parts = [];
    const re = /([.#]?[\w-]+)|\[([\w-]+)(?:([~|^$*]?=)"?([^"\]]*)"?)?\]/g;
    let m;
    while ((m = re.exec(term))) {
      if (m[1]) {
        if (m[1][0] === '.') parts.push({ kind: 'class', value: m[1].slice(1) });
        else if (m[1][0] === '#') parts.push({ kind: 'id', value: m[1].slice(1) });
        else parts.push({ kind: 'tag', value: m[1].toLowerCase() });
      } else if (m[2]) {
        parts.push({ kind: 'attr', name: m[2], op: m[3] || null, value: m[4] });
      }
    }
    return parts;
  }).filter((parts) => parts.length);
}

class Node {
  constructor(tag) {
    this.tag = String(tag || 'div').toLowerCase();
    this.className = '';
    this.attributes = new Map();
    this.children = [];
    this.parentNode = null;
    this.hidden = false;
    this._text = '';
    const self = this;
    this.dataset = new Proxy({}, {
      get(_t, key) {
        if (typeof key !== 'string') return undefined;
        const attr = 'data-' + key.replace(/[A-Z]/g, (c) => '-' + c.toLowerCase());
        return self.attributes.has(attr) ? self.attributes.get(attr) : undefined;
      },
      set(_t, key, value) {
        const attr = 'data-' + String(key).replace(/[A-Z]/g, (c) => '-' + c.toLowerCase());
        self.attributes.set(attr, String(value));
        return true;
      },
      has(_t, key) {
        const attr = 'data-' + String(key).replace(/[A-Z]/g, (c) => '-' + c.toLowerCase());
        return self.attributes.has(attr);
      },
    });
    this.style = { setProperty() {}, removeProperty() {}, getPropertyValue() { return ''; }, display: '' };
    this.listeners = new Map();
  }

  get classList() {
    const el = this;
    return {
      add(name) { const s = new Set(el.className.split(/\s+/).filter(Boolean)); s.add(name); el.className = [...s].join(' '); },
      remove(name) { el.className = el.className.split(/\s+/).filter((v) => v && v !== name).join(' '); },
      contains(name) { return el.className.split(/\s+/).indexOf(name) >= 0; },
      toggle(name, force) { if (force === undefined ? this.contains(name) : !force) this.remove(name); else this.add(name); },
    };
  }

  setAttribute(name, value) {
    this.attributes.set(name, String(value));
    if (name === 'class') this.className = String(value);
    if (name === 'hidden') this.hidden = true;
  }
  getAttribute(name) {
    if (name === 'class') return this.className || null;
    return this.attributes.has(name) ? this.attributes.get(name) : null;
  }
  appendChild(child) { child.parentNode = this; this.children.push(child); return child; }
  addEventListener(type, cb) { this.listeners.set(type, cb); }
  removeEventListener(type) { this.listeners.delete(type); }

  set textContent(value) { this._text = String(value); this.children = []; }
  get textContent() {
    if (this.children.length) return this.children.map((c) => c.textContent).join('');
    return this._text;
  }

  _matchesTerm(parts) {
    return parts.every((p) => {
      if (p.kind === 'class') return this.classList.contains(p.value);
      if (p.kind === 'tag') return this.tag === p.value;
      if (p.kind === 'id') return this.getAttribute('id') === p.value;
      if (p.kind === 'attr') {
        if (!this.attributes.has(p.name)) return false;
        if (!p.op) return true;
        return this.attributes.get(p.name) === p.value;
      }
      return false;
    });
  }
  matches(selector) { return parseSelector(selector).some((parts) => this._matchesTerm(parts)); }

  _walk(out) { for (const c of this.children) { out.push(c); c._walk(out); } return out; }
  querySelectorAll(selector) {
    const groups = parseSelector(selector);
    return this._walk([]).filter((node) => groups.some((parts) => node._matchesTerm(parts)));
  }
  querySelector(selector) { const all = this.querySelectorAll(selector); return all.length ? all[0] : null; }
  closest(selector) {
    const groups = parseSelector(selector);
    let node = this;
    while (node) { if (groups.some((parts) => node._matchesTerm(parts))) return node; node = node.parentNode; }
    return null;
  }
}

function el(tag, className, attrs, text) {
  const node = new Node(tag);
  if (className) node.setAttribute('class', className);
  if (attrs) Object.keys(attrs).forEach((k) => node.setAttribute(k, attrs[k]));
  if (text !== undefined) node.textContent = text;
  return node;
}

// A role header (avatar "H" + "Hermes") — the source of the "HHermes" leak.
function roleHeader() {
  const role = el('div', 'msg-role assistant');
  role.appendChild(el('div', 'role-icon assistant', null, 'H'));
  role.appendChild(el('span', 'msg-role-name', null, 'Hermes'));
  return role;
}
// The worklog "Processed 13s" chip — the source of the "Processed 13s" leak.
function worklogGroup() {
  const group = el('div', 'tool-worklog-group tool-call-group', { 'data-anchor-scene-owner': '1' });
  const summary = el('button', 'tool-call-group-summary tool-worklog-summary');
  summary.appendChild(el('span', 'tool-call-group-label', null, 'Processed'));
  summary.appendChild(el('span', 'tool-call-group-duration', null, ' 13s'));
  group.appendChild(summary);
  const body = el('div', 'tool-call-group-body tool-worklog-body', { hidden: 'hidden' });
  body.appendChild(el('div', 'wl-reason', null, 'internal tool reasoning that must never be spoken'));
  group.appendChild(body);
  return group;
}
function answerSegment(rawText, bodyText, opts) {
  const seg = el('div', 'assistant-segment', { 'data-msg-idx': String((opts && opts.idx) || 1) });
  if (rawText !== null && rawText !== undefined) seg.setAttribute('data-raw-text', rawText);
  if (opts && opts.live) seg.setAttribute('data-live-assistant', '1');
  // Reasoning rendered INSIDE the segment uses .thinking-card-body, but pin the
  // chrome exclusion by nesting a stray .msg-body inside a thinking card too.
  if (opts && opts.reasoning) {
    const card = el('div', 'thinking-card');
    card.appendChild(el('div', 'msg-body', null, 'REASONING: ' + opts.reasoning));
    seg.appendChild(card);
  }
  if (bodyText !== null && bodyText !== undefined) seg.appendChild(el('div', 'msg-body', null, bodyText));
  return seg;
}
function assistantTurn(segments, opts) {
  const turn = el('div', 'msg-row assistant-turn', { 'data-role': 'assistant', 'data-session-id': 'session-1' });
  if (opts && opts.live) turn.setAttribute('id', 'liveAssistantTurn');
  turn.appendChild(roleHeader());
  const blocks = el('div', 'assistant-turn-blocks');
  if (opts && opts.worklog) blocks.appendChild(worklogGroup());
  segments.forEach((s) => blocks.appendChild(s));
  turn.appendChild(blocks);
  return turn;
}

// ── vm harness ──────────────────────────────────────────────────────────────
const documentRoot = el('div', 'msgInner');
function makeControl(id) {
  const node = new Node('div');
  node.setAttribute('id', id);
  return node;
}
const controls = {
  btnVoiceMode: makeControl('btnVoiceMode'),
  voiceModeBar: makeControl('voiceModeBar'),
  voiceModeIndicator: makeControl('voiceModeIndicator'),
  voiceModeLabel: makeControl('voiceModeLabel'),
  msg: makeControl('msg'),
};

const documentStub = {
  baseURI: 'https://chat.test/',
  body: new Node('body'),
  getElementById: (id) => controls[id] || (id === 'liveAssistantTurn' ? documentRoot.querySelector('#liveAssistantTurn') : null),
  querySelectorAll: (selector) => documentRoot.querySelectorAll(selector),
  querySelector: (selector) => documentRoot.querySelector(selector),
  createElement: (tag) => new Node(tag),
  addEventListener() {},
  removeEventListener() {},
};

const sandbox = {
  console,
  window: null,
  document: documentStub,
  navigator: { mediaDevices: { getSupportedConstraints: () => ({}), getUserMedia: async () => ({ getTracks: () => [], getAudioTracks: () => [] }) } },
  MediaRecorder: function MediaRecorder() {},
  matchMedia: () => ({ matches: false, addEventListener() {}, removeEventListener() {} }),
  localStorage: { getItem: () => null, setItem() {}, removeItem() {} },
  crypto: { getRandomValues(b) { for (let i = 0; i < b.length; i += 1) b[i] = i + 1; return b; } },
  location: { protocol: 'https:', host: 'chat.test', href: 'https://chat.test/' },
  fetch: () => Promise.reject(new Error('no network in probe')),
  setTimeout: () => 0,
  clearTimeout: () => {},
  setInterval: () => 0,
  clearInterval: () => {},
  queueMicrotask: (fn) => Promise.resolve().then(fn),
  S: { session: { session_id: 'session-1' }, busy: false, activeStreamId: null },
  URL,
  Math, JSON, String, Number, Boolean, Error, Array, Object, Set, Map,
  parseInt, parseFloat, isNaN, Date,
};
sandbox.window = sandbox;
sandbox.globalThis = sandbox;
sandbox.window.matchMedia = sandbox.matchMedia;
sandbox.window.MediaRecorder = sandbox.MediaRecorder;

vm.createContext(sandbox);
vm.runInContext(SOURCE, sandbox, { filename: 'atlas-voice.js' });

const internals = sandbox.window.__atlasVoiceInternals;
if (!internals) {
  throw new Error('atlas-voice.js did not expose __atlasVoiceInternals (early return? missing DOM stub)');
}

function setTurn(turn) {
  documentRoot.children = [];
  if (turn) documentRoot.appendChild(turn);
}

// Mirror pumpAssistantResponse's one-ahead consume loop over a full reply: a
// partial pass (isFinal=false) then the completion pass (isFinal=true) + tail
// flush. Proves every sentence is chunked, not just sentence one.
function pumpChunks(text) {
  let consumed = 0;
  let first = true;
  const chunks = [];
  for (let pass = 0; pass < 12; pass += 1) {
    const remaining = text.slice(consumed).replace(/^\s+/, '');
    const skipped = text.slice(consumed).length - remaining.length;
    const ex = internals.adaptiveChunks(remaining, false, first);
    if (!ex.chunks.length) break;
    first = false;
    consumed += skipped + ex.consumed;
    chunks.push(...ex.chunks);
  }
  // completion pass
  {
    const remaining = text.slice(consumed).replace(/^\s+/, '');
    const skipped = text.slice(consumed).length - remaining.length;
    const ex = internals.adaptiveChunks(remaining, true, first);
    if (ex.chunks.length) { consumed += skipped + ex.consumed; chunks.push(...ex.chunks); }
    const tail = text.slice(consumed).trim();
    if (tail) { chunks.push(tail); consumed = text.length; }
  }
  return { chunks, consumed, covered: chunks.join(' ') };
}

const results = {};

// (1) Finalized multi-segment turn: caption/TTS is answer-only, all sentences chunked.
{
  const answer = 'Sentence one is here. Sentence two follows it. And sentence three concludes.';
  const turn = assistantTurn([
    answerSegment('earlier interim answer', 'earlier interim answer', { idx: 0 }),
    answerSegment(answer, answer, { idx: 1, reasoning: 'let me think' }),
  ], { worklog: true });
  // fold the interim segment into the worklog exactly as the settle renderer does
  const interim = turn.querySelectorAll('.assistant-segment')[0];
  interim.setAttribute('class', 'assistant-segment assistant-segment-worklog-source');
  interim.setAttribute('aria-hidden', 'true');
  interim.hidden = true;
  setTurn(turn);
  const extracted = internals.collectAssistantResponse();
  const pumped = pumpChunks(extracted.text);
  results.finalized_multi_segment = {
    text: extracted.text,
    error: extracted.error,
    leaksAvatar: /HHermes|Hermes/.test(extracted.text),
    leaksProcessed: /Processed|13s/.test(extracted.text),
    leaksReasoning: /REASONING/.test(extracted.text),
    leaksInterim: /interim/.test(extracted.text),
    chunkCount: pumped.chunks.length,
    chunkConsumedAll: pumped.consumed === extracted.text.length,
    sentenceOnePresent: /one/.test(pumped.covered),
    sentenceTwoPresent: /two/.test(pumped.covered),
    sentenceThreePresent: /three/.test(pumped.covered),
  };
}

// (2) Pre-settle worklog-only frame (the STREAM_DONE-beats-settle race): the
// answer segment is not rendered yet — role header + "Processed 13s" only.
// Must extract to '' so the final pump RETRIES instead of speaking chrome.
{
  const turn = assistantTurn([], { worklog: true });
  setTurn(turn);
  const extracted = internals.collectAssistantResponse();
  results.presettle_worklog_only = {
    text: extracted.text,
    isEmpty: extracted.text === '',
    error: extracted.error,
  };
}

// (3) Live streaming segment (no data-raw-text yet) — read via .msg-body.
{
  const partial = 'The reply is still streaming right now';
  const turn = assistantTurn([answerSegment(null, partial, { idx: 1, live: true })], { worklog: false, live: true });
  setTurn(turn);
  const extracted = internals.collectAssistantResponse();
  results.live_streaming_reads_body = {
    text: extracted.text,
    matches: extracted.text === partial,
    leaksAvatar: /Hermes/.test(extracted.text),
  };
}

// (4) Reasoning + worklog chrome nested with stray .msg-body must be excluded.
{
  const answer = 'Only this answer body should be spoken aloud.';
  const seg = answerSegment(null, answer, { idx: 1, reasoning: 'hidden chain of thought' });
  const turn = assistantTurn([seg], { worklog: true });
  setTurn(turn);
  const extracted = internals.collectAssistantResponse();
  results.chrome_excluded = {
    text: extracted.text,
    matches: extracted.text === answer,
    leaksReasoning: /REASONING|chain of thought/.test(extracted.text),
  };
}

// (5) Streaming growth then finalize: extraction grows monotonically to the full
// reply — the queue keeps receiving sentences until the turn is final.
{
  const two = 'First sentence. Second sentence.';
  const four = 'First sentence. Second sentence. Third sentence. Fourth sentence.';
  const liveTurn = assistantTurn([answerSegment(null, two, { idx: 1, live: true })], { live: true });
  setTurn(liveTurn);
  const partial = internals.collectAssistantResponse().text;
  const settledTurn = assistantTurn([answerSegment(four, four, { idx: 1 })], { worklog: true });
  setTurn(settledTurn);
  const full = internals.collectAssistantResponse().text;
  const pumped = pumpChunks(full);
  results.streaming_growth = {
    partial,
    full,
    grows: full.length > partial.length && full.startsWith(partial),
    allFourChunked: /First/.test(pumped.covered) && /Second/.test(pumped.covered) && /Third/.test(pumped.covered) && /Fourth/.test(pumped.covered),
    consumedAll: pumped.consumed === full.length,
  };
}

// (6) A turn with a recovered/transient error segment AND a real answer must
// speak the answer (error must NOT veto a turn that produced content).
{
  const answer = 'Here is the real answer despite a transient tool hiccup.';
  const errSeg = answerSegment(null, 'a tool call failed transiently', { idx: 1 });
  errSeg.setAttribute('data-error', '1');
  const turn = assistantTurn([errSeg, answerSegment(answer, answer, { idx: 2 })], { worklog: false });
  setTurn(turn);
  const extracted = internals.collectAssistantResponse();
  results.error_segment_with_answer = {
    text: extracted.text,
    error: extracted.error,
    speaksAnswer: extracted.text === answer,
  };
}

// (7) A turn that is ONLY an error envelope (no answer) still reports error.
{
  const errSeg = answerSegment(null, 'The provider returned an error.', { idx: 1 });
  errSeg.setAttribute('data-error', '1');
  const turn = assistantTurn([errSeg], { worklog: false });
  setTurn(turn);
  const extracted = internals.collectAssistantResponse();
  results.error_only_reports_error = {
    text: extracted.text,
    isEmpty: extracted.text === '',
    error: extracted.error,
  };
}

process.stdout.write(JSON.stringify(results, null, 2) + '\n');