media: auto-heal stale upstream av epoch
This commit is contained in:
parent
555f88928b
commit
a75463669f
6
Cargo.lock
generated
6
Cargo.lock
generated
@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
||||
|
||||
[[package]]
|
||||
name = "lesavka_client"
|
||||
version = "0.21.9"
|
||||
version = "0.21.10"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-stream",
|
||||
@ -1686,7 +1686,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lesavka_common"
|
||||
version = "0.21.9"
|
||||
version = "0.21.10"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
@ -1698,7 +1698,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lesavka_server"
|
||||
version = "0.21.9"
|
||||
version = "0.21.10"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
|
||||
@ -4,7 +4,7 @@ path = "src/main.rs"
|
||||
|
||||
[package]
|
||||
name = "lesavka_client"
|
||||
version = "0.21.9"
|
||||
version = "0.21.10"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
|
||||
@ -120,6 +120,34 @@ use super::*;
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Keeps `startup_epoch_auto_heal_defaults_to_one_safe_delayed_recovery` explicit because the first Google Meet join can latch a stale UAC epoch.
|
||||
/// Inputs are optional env-style values; output is the parsed recovery delay.
|
||||
fn startup_epoch_auto_heal_defaults_to_one_safe_delayed_recovery() {
|
||||
assert_eq!(
|
||||
parse_upstream_epoch_auto_heal_delay(None, None),
|
||||
Some(Duration::from_millis(DEFAULT_UPSTREAM_AUTO_HEAL_AFTER_MS))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_upstream_epoch_auto_heal_delay(Some("1"), Some("12000")),
|
||||
Some(Duration::from_millis(12_000))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_upstream_epoch_auto_heal_delay(Some("true"), Some("0")),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Keeps `startup_epoch_auto_heal_can_be_disabled_for_lab_runs` explicit because recovery should be operator-controllable during calibration.
|
||||
/// Inputs are disable spellings; output is no scheduled recovery.
|
||||
fn startup_epoch_auto_heal_can_be_disabled_for_lab_runs() {
|
||||
for raw in ["0", "false", "off", "no", " FALSE "] {
|
||||
assert_eq!(parse_upstream_epoch_auto_heal_delay(Some(raw), None), None);
|
||||
}
|
||||
assert!(!disables_upstream_epoch_auto_heal("yes"));
|
||||
}
|
||||
|
||||
/// Verifies the live uplink queue emits one physically bundled HEVC frame and PCM span.
|
||||
///
|
||||
/// Inputs: pre-stamped HEVC video plus two nearby audio packets, exactly as
|
||||
|
||||
@ -15,6 +15,7 @@ impl LesavkaClientApp {
|
||||
media_controls: crate::live_media_control::LiveMediaControls,
|
||||
) {
|
||||
let mut delay = Duration::from_secs(1);
|
||||
let mut startup_epoch_heal_delay = upstream_epoch_auto_heal_delay();
|
||||
static FAIL_CNT: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
loop {
|
||||
@ -256,6 +257,9 @@ impl LesavkaClientApp {
|
||||
delay = Duration::from_secs(1);
|
||||
camera_telemetry.record_connected();
|
||||
microphone_telemetry.record_connected();
|
||||
if let Some(heal_delay) = startup_epoch_heal_delay.take() {
|
||||
spawn_upstream_epoch_auto_heal(ep.clone(), heal_delay);
|
||||
}
|
||||
while resp.get_mut().message().await.transpose().is_some() {}
|
||||
camera_telemetry.record_disconnect("bundled webcam media stream ended");
|
||||
microphone_telemetry.record_disconnect("bundled webcam media stream ended");
|
||||
@ -293,3 +297,77 @@ impl LesavkaClientApp {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_UPSTREAM_AUTO_HEAL_AFTER_MS: u64 = 3_000;
|
||||
|
||||
/// Resolve whether the live bundled uplink should force one startup epoch heal.
|
||||
///
|
||||
/// Inputs: `LESAVKA_UPSTREAM_AUTO_HEAL` and
|
||||
/// `LESAVKA_UPSTREAM_AUTO_HEAL_AFTER_MS`; output is the delay before the heal
|
||||
/// request, or `None` when disabled. Why: browser/WebRTC consumers can latch a
|
||||
/// stale UAC epoch on first join, and the safe fix is to retire the mic epoch
|
||||
/// once the live bundled stream has settled rather than changing calibration.
|
||||
#[cfg(not(coverage))]
|
||||
fn upstream_epoch_auto_heal_delay() -> Option<Duration> {
|
||||
parse_upstream_epoch_auto_heal_delay(
|
||||
std::env::var("LESAVKA_UPSTREAM_AUTO_HEAL").ok().as_deref(),
|
||||
std::env::var("LESAVKA_UPSTREAM_AUTO_HEAL_AFTER_MS")
|
||||
.ok()
|
||||
.as_deref(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Parse the live-uplink auto-heal knobs without reading global environment.
|
||||
///
|
||||
/// Inputs are optional raw environment values. Output is a bounded delay, or
|
||||
/// `None` when the operator opted out. Why: the default needs to be safe enough
|
||||
/// for real calls while tests can pin the behavior without racing process env.
|
||||
#[cfg(not(coverage))]
|
||||
fn parse_upstream_epoch_auto_heal_delay(
|
||||
enabled: Option<&str>,
|
||||
delay_ms: Option<&str>,
|
||||
) -> Option<Duration> {
|
||||
if enabled.is_some_and(disables_upstream_epoch_auto_heal) {
|
||||
return None;
|
||||
}
|
||||
let millis = delay_ms
|
||||
.and_then(|raw| raw.trim().parse::<u64>().ok())
|
||||
.unwrap_or(DEFAULT_UPSTREAM_AUTO_HEAL_AFTER_MS);
|
||||
(millis > 0).then(|| Duration::from_millis(millis))
|
||||
}
|
||||
|
||||
/// Return whether a text env value disables startup A/V epoch healing.
|
||||
#[cfg(not(coverage))]
|
||||
fn disables_upstream_epoch_auto_heal(raw: &str) -> bool {
|
||||
matches!(
|
||||
raw.trim().to_ascii_lowercase().as_str(),
|
||||
"0" | "false" | "off" | "no"
|
||||
)
|
||||
}
|
||||
|
||||
/// Ask the relay to retire the current UAC epoch after the stream is live.
|
||||
///
|
||||
/// Inputs: the connected relay channel and a delay. Output is only log
|
||||
/// telemetry. Why: this mimics the known-good probe side effect that superseded
|
||||
/// a stale first-join epoch and forced the live client stream to reconnect
|
||||
/// cleanly without touching USB enumeration or A/V calibration.
|
||||
#[cfg(not(coverage))]
|
||||
fn spawn_upstream_epoch_auto_heal(ep: Channel, delay: Duration) {
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(delay).await;
|
||||
let mut client = RelayClient::new(ep);
|
||||
match client.recover_uac(Request::new(Empty {})).await {
|
||||
Ok(reply) => {
|
||||
if reply.into_inner().ok {
|
||||
info!(
|
||||
delay_ms = delay.as_millis(),
|
||||
"📦🩺 automatic upstream A/V epoch heal requested"
|
||||
);
|
||||
} else {
|
||||
warn!("📦🩺 automatic upstream A/V epoch heal was refused by relay");
|
||||
}
|
||||
}
|
||||
Err(err) => warn!(%err, "📦🩺 automatic upstream A/V epoch heal failed"),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@ -50,7 +50,9 @@ impl CommandKind {
|
||||
"on" | "force-on" => Some(Self::On),
|
||||
"off" | "force-off" => Some(Self::Off),
|
||||
"recover-usb" => Some(Self::RecoverUsb),
|
||||
"recover-uac" => Some(Self::RecoverUac),
|
||||
"recover-uac" | "heal-av" | "heal-upstream" | "recover-upstream" => {
|
||||
Some(Self::RecoverUac)
|
||||
}
|
||||
"recover-uvc" => Some(Self::RecoverUvc),
|
||||
"reset-usb" | "hard-reset-usb" => Some(Self::ResetUsb),
|
||||
"upstream-sync" | "sync" => Some(Self::UpstreamSync),
|
||||
@ -97,7 +99,7 @@ enum ParseOutcome {
|
||||
}
|
||||
|
||||
fn usage() -> &'static str {
|
||||
"Usage: lesavka-relayctl [--server http://HOST:50051] <status|version|upstream-sync|output-delay-probe [duration_s warmup_s period_ms width_ms codes audio_delay_us video_delay_us]|calibration|calibrate <audio_delta_us> <video_delta_us> [note]|calibration-save-default|calibration-restore-default|calibration-restore-factory|auto|on|off|recover-usb|recover-uac|recover-uvc|reset-usb>"
|
||||
"Usage: lesavka-relayctl [--server http://HOST:50051] <status|version|upstream-sync|output-delay-probe [duration_s warmup_s period_ms width_ms codes audio_delay_us video_delay_us]|calibration|calibrate <audio_delta_us> <video_delta_us> [note]|calibration-save-default|calibration-restore-default|calibration-restore-factory|auto|on|off|recover-usb|recover-uac|heal-av|recover-uvc|reset-usb>"
|
||||
}
|
||||
|
||||
/// Keeps `parse_args_outcome_from` explicit because it sits on CLI orchestration, where operators need deterministic exits and artifact paths.
|
||||
|
||||
@ -54,6 +54,11 @@ fn command_aliases_parse_to_stable_actions() {
|
||||
CommandKind::parse("recover-uac"),
|
||||
Some(CommandKind::RecoverUac)
|
||||
);
|
||||
assert_eq!(CommandKind::parse("heal-av"), Some(CommandKind::RecoverUac));
|
||||
assert_eq!(
|
||||
CommandKind::parse("heal-upstream"),
|
||||
Some(CommandKind::RecoverUac)
|
||||
);
|
||||
assert_eq!(
|
||||
CommandKind::parse("recover-uvc"),
|
||||
Some(CommandKind::RecoverUvc)
|
||||
|
||||
@ -202,7 +202,7 @@
|
||||
let server_addr = selected_server_addr(&server_entry, server_addr_fallback.as_ref());
|
||||
widgets_for_click
|
||||
.status_label
|
||||
.set_text("Recover UAC 1/3: asking microphone stream to stand down cleanly...");
|
||||
.set_text("Heal A/V 1/3: retiring the stale upstream media epoch cleanly...");
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let result = recover_uac_soft(&server_addr).map_err(|err| format!("{err:#}"));
|
||||
@ -212,20 +212,20 @@
|
||||
glib::timeout_add_local(Duration::from_millis(100), move || match rx.try_recv() {
|
||||
Ok(Ok(())) => {
|
||||
widgets.status_label.set_text(
|
||||
"Recover UAC 2/3: old mic sink released. Recover UAC 3/3: client will reconnect without resetting USB.",
|
||||
"Heal A/V 2/3: old epoch released. Heal A/V 3/3: client will reconnect without resetting USB or calibration.",
|
||||
);
|
||||
glib::ControlFlow::Break
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
widgets
|
||||
.status_label
|
||||
.set_text(&format!("Recover UAC failed: {err}"));
|
||||
.set_text(&format!("Heal A/V failed: {err}"));
|
||||
glib::ControlFlow::Break
|
||||
}
|
||||
Err(std::sync::mpsc::TryRecvError::Empty) => glib::ControlFlow::Continue,
|
||||
Err(std::sync::mpsc::TryRecvError::Disconnected) => {
|
||||
widgets.status_label.set_text(
|
||||
"Recover UAC failed: relay stopped responding before completion.",
|
||||
"Heal A/V failed: relay stopped responding before completion.",
|
||||
);
|
||||
glib::ControlFlow::Break
|
||||
}
|
||||
|
||||
@ -43,8 +43,8 @@
|
||||
"Soft recovery: reopen HID and verify enumeration without detaching the USB gadget.",
|
||||
);
|
||||
let uac_recover_button = rail_button(
|
||||
"UAC",
|
||||
"Soft recovery: retire the active mic sink so the audio stream reconnects without resetting USB.",
|
||||
"Heal A/V",
|
||||
"Soft recovery: retire the stale upstream A/V epoch so bundled mic+camera reconnect without resetting USB or changing calibration.",
|
||||
);
|
||||
let uvc_recover_button = rail_button(
|
||||
"UVC",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lesavka_common"
|
||||
version = "0.21.9"
|
||||
version = "0.21.10"
|
||||
edition = "2024"
|
||||
build = "build.rs"
|
||||
|
||||
|
||||
319
scripts/manual/run_google_meet_observer_probe.sh
Executable file
319
scripts/manual/run_google_meet_observer_probe.sh
Executable file
@ -0,0 +1,319 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/manual/run_google_meet_observer_probe.sh
|
||||
# Manual: Google Meet observer-path probe; not part of CI.
|
||||
#
|
||||
# Why this exists: Google Meet blocks reliable unattended automation, but we
|
||||
# still need repeatable evidence about whether distortion appears after raw
|
||||
# Lesavka UVC/UAC output. This harness leaves Meet joining and recording to a
|
||||
# human operator, then injects the same synthetic bundled media used by the
|
||||
# client-to-RCT probe and analyzes the observer-side recording when provided.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
||||
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"
|
||||
|
||||
LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-https://38.28.125.112:50051}
|
||||
LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}
|
||||
LESAVKA_GOOGLE_MEET_URL=${LESAVKA_GOOGLE_MEET_URL:-}
|
||||
LESAVKA_MEET_OPEN_URL=${LESAVKA_MEET_OPEN_URL:-0}
|
||||
LESAVKA_MEET_SKIP_PROMPTS=${LESAVKA_MEET_SKIP_PROMPTS:-0}
|
||||
LESAVKA_MEET_START_DELAY_SECONDS=${LESAVKA_MEET_START_DELAY_SECONDS:-0}
|
||||
LESAVKA_MEET_OBSERVER_CAPTURE=${LESAVKA_MEET_OBSERVER_CAPTURE:-}
|
||||
LESAVKA_MEET_ANALYSIS_REQUIRED=${LESAVKA_MEET_ANALYSIS_REQUIRED:-0}
|
||||
LESAVKA_MEET_LOCAL_REVIEW=${LESAVKA_MEET_LOCAL_REVIEW:-1}
|
||||
|
||||
PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-20}
|
||||
PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4}
|
||||
PROBE_PULSE_PERIOD_MS=${PROBE_PULSE_PERIOD_MS:-1000}
|
||||
PROBE_PULSE_WIDTH_MS=${PROBE_PULSE_WIDTH_MS:-120}
|
||||
PROBE_MARKER_TICK_PERIOD=${PROBE_MARKER_TICK_PERIOD:-5}
|
||||
PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}
|
||||
|
||||
LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
ARTIFACT_DIR="${LOCAL_OUTPUT_DIR%/}/lesavka-google-meet-observer-probe-${STAMP}"
|
||||
RUN_LOG="${ARTIFACT_DIR}/google-meet-observer-run.log"
|
||||
VERSION_TXT="${ARTIFACT_DIR}/lesavka-versions.txt"
|
||||
OPERATOR_CHECKLIST="${ARTIFACT_DIR}/operator-checklist.txt"
|
||||
MANUAL_TIMING_JSON="${ARTIFACT_DIR}/manual-timing.json"
|
||||
CLIENT_TIMELINE_JSON="${ARTIFACT_DIR}/client-transport-timeline.json"
|
||||
OBSERVER_CAPTURE_LOCAL=""
|
||||
ANALYSIS_DIR="${ARTIFACT_DIR}/observer-analysis"
|
||||
|
||||
mkdir -p "${ARTIFACT_DIR}"
|
||||
exec > >(tee -a "${RUN_LOG}") 2>&1
|
||||
|
||||
json_now_ns() {
|
||||
python3 - <<'PY'
|
||||
import time
|
||||
print(time.time_ns())
|
||||
PY
|
||||
}
|
||||
|
||||
prompt_or_sleep() {
|
||||
local message=$1
|
||||
local default_sleep=$2
|
||||
printf '\a'
|
||||
if [[ "${LESAVKA_MEET_SKIP_PROMPTS}" == "1" ]]; then
|
||||
echo "==> ${message}; sleeping ${default_sleep}s because prompts are disabled"
|
||||
sleep "${default_sleep}"
|
||||
return 0
|
||||
fi
|
||||
if [[ -t 0 ]]; then
|
||||
echo
|
||||
echo "==> ${message}"
|
||||
read -r -p "Press Enter when ready to continue..." _
|
||||
else
|
||||
echo "==> ${message}; stdin is not interactive, sleeping ${default_sleep}s"
|
||||
sleep "${default_sleep}"
|
||||
fi
|
||||
}
|
||||
|
||||
write_operator_checklist() {
|
||||
cat >"${OPERATOR_CHECKLIST}" <<EOF
|
||||
Google Meet observer probe checklist
|
||||
|
||||
Goal:
|
||||
Determine whether audio/video corruption starts inside Google Meet/WebRTC,
|
||||
after raw Lesavka UVC/UAC output has already been proven clean.
|
||||
|
||||
Manual setup:
|
||||
1. On the RCT/Tethys browser, join the Meet using Lesavka Camera and Lesavka Microphone.
|
||||
2. On this workstation, join the same Meet as an observer: muted mic, camera off.
|
||||
3. Pin or spotlight the RCT/Lesavka participant so the synthetic flash video is large.
|
||||
4. Disable captions and any visual overlays that could cover the tile.
|
||||
5. Start an observer-side recording that captures both Meet video and Meet audio.
|
||||
6. Return to this terminal and continue the probe.
|
||||
|
||||
Preferred observer recording:
|
||||
A local screen/tab recording from this workstation is best. Google cloud
|
||||
recording is usable, but it adds another processing layer.
|
||||
|
||||
Analysis:
|
||||
If LESAVKA_MEET_OBSERVER_CAPTURE is set, this script copies and analyzes it.
|
||||
Otherwise, rerun or invoke lesavka-sync-analyze manually against the recording.
|
||||
|
||||
Meet URL:
|
||||
${LESAVKA_GOOGLE_MEET_URL:-not provided}
|
||||
EOF
|
||||
}
|
||||
|
||||
build_probe_tools() {
|
||||
echo "==> prebuilding Meet observer probe tools"
|
||||
(
|
||||
cd "${REPO_ROOT}"
|
||||
cargo build -p lesavka_client \
|
||||
--bin lesavka-sync-probe \
|
||||
--bin lesavka-sync-analyze \
|
||||
--bin lesavka-relayctl
|
||||
)
|
||||
}
|
||||
|
||||
print_versions() {
|
||||
echo "==> Lesavka versions under test"
|
||||
local status=0
|
||||
(
|
||||
cd "${REPO_ROOT}"
|
||||
LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \
|
||||
"${REPO_ROOT}/target/debug/lesavka-relayctl" \
|
||||
--server "${LESAVKA_SERVER_ADDR}" \
|
||||
version
|
||||
) >"${VERSION_TXT}" 2>&1 || status=$?
|
||||
sed 's/^/ ↪ /' "${VERSION_TXT}" || true
|
||||
return "${status}"
|
||||
}
|
||||
|
||||
maybe_open_meet_url() {
|
||||
if [[ -z "${LESAVKA_GOOGLE_MEET_URL}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
echo "==> Meet URL: ${LESAVKA_GOOGLE_MEET_URL}"
|
||||
if [[ "${LESAVKA_MEET_OPEN_URL}" == "1" ]]; then
|
||||
xdg-open "${LESAVKA_GOOGLE_MEET_URL}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
validate_start_delay() {
|
||||
if ! [[ "${LESAVKA_MEET_START_DELAY_SECONDS}" =~ ^[0-9]+$ ]]; then
|
||||
echo "LESAVKA_MEET_START_DELAY_SECONDS must be a non-negative integer" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
run_synthetic_probe() {
|
||||
echo "==> running synthetic bundled media through Google Meet path"
|
||||
local probe_started_ns probe_finished_ns
|
||||
probe_started_ns="$(json_now_ns)"
|
||||
(
|
||||
cd "${REPO_ROOT}"
|
||||
LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \
|
||||
"${REPO_ROOT}/target/debug/lesavka-sync-probe" \
|
||||
--server "${LESAVKA_SERVER_ADDR}" \
|
||||
--duration-seconds "${PROBE_DURATION_SECONDS}" \
|
||||
--warmup-seconds "${PROBE_WARMUP_SECONDS}" \
|
||||
--pulse-period-ms "${PROBE_PULSE_PERIOD_MS}" \
|
||||
--pulse-width-ms "${PROBE_PULSE_WIDTH_MS}" \
|
||||
--marker-tick-period "${PROBE_MARKER_TICK_PERIOD}" \
|
||||
--event-width-codes "${PROBE_EVENT_WIDTH_CODES}" \
|
||||
--timeline-json "${CLIENT_TIMELINE_JSON}"
|
||||
)
|
||||
probe_finished_ns="$(json_now_ns)"
|
||||
python3 - "${MANUAL_TIMING_JSON}" "${probe_started_ns}" "${probe_finished_ns}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
|
||||
path = sys.argv[1]
|
||||
payload = {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
payload.update({
|
||||
"probe_started_unix_ns": int(sys.argv[2]),
|
||||
"probe_finished_unix_ns": int(sys.argv[3]),
|
||||
"timing_note": (
|
||||
"Observer capture times are operator-confirmed, so freshness from a "
|
||||
"manual recording is approximate unless the recorder supplies its own "
|
||||
"wall-clock start timestamp."
|
||||
),
|
||||
"updated_at_unix_ns": time.time_ns(),
|
||||
})
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
PY
|
||||
}
|
||||
|
||||
record_operator_ready_time() {
|
||||
local ready_ns
|
||||
ready_ns="$(json_now_ns)"
|
||||
python3 - "${MANUAL_TIMING_JSON}" "${ready_ns}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
|
||||
path = sys.argv[1]
|
||||
ready_ns = int(sys.argv[2])
|
||||
payload = {
|
||||
"observer_recording_confirmed_unix_ns": ready_ns,
|
||||
"created_at_unix_ns": time.time_ns(),
|
||||
}
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
PY
|
||||
}
|
||||
|
||||
copy_observer_capture_if_present() {
|
||||
if [[ -z "${LESAVKA_MEET_OBSERVER_CAPTURE}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if [[ ! -f "${LESAVKA_MEET_OBSERVER_CAPTURE}" ]]; then
|
||||
echo "observer recording not found: ${LESAVKA_MEET_OBSERVER_CAPTURE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
local ext
|
||||
ext="${LESAVKA_MEET_OBSERVER_CAPTURE##*.}"
|
||||
if [[ "${ext}" == "${LESAVKA_MEET_OBSERVER_CAPTURE}" ]]; then
|
||||
ext="mkv"
|
||||
fi
|
||||
OBSERVER_CAPTURE_LOCAL="${ARTIFACT_DIR}/observer-google-meet-capture.${ext}"
|
||||
cp "${LESAVKA_MEET_OBSERVER_CAPTURE}" "${OBSERVER_CAPTURE_LOCAL}"
|
||||
echo " ↪ observer_capture=${OBSERVER_CAPTURE_LOCAL}"
|
||||
}
|
||||
|
||||
prompt_for_observer_capture_path() {
|
||||
if [[ -n "${LESAVKA_MEET_OBSERVER_CAPTURE}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if [[ "${LESAVKA_MEET_SKIP_PROMPTS}" == "1" || ! -t 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
echo
|
||||
read -r -p "Path to observer recording for analysis (blank to skip): " LESAVKA_MEET_OBSERVER_CAPTURE
|
||||
}
|
||||
|
||||
analyze_observer_capture() {
|
||||
if [[ -z "${OBSERVER_CAPTURE_LOCAL}" ]]; then
|
||||
echo "==> observer capture analysis skipped"
|
||||
echo " ↪ set LESAVKA_MEET_OBSERVER_CAPTURE=/path/to/recording.webm and rerun analysis"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "==> analyzing observer-side Meet recording"
|
||||
mkdir -p "${ANALYSIS_DIR}"
|
||||
local status=0
|
||||
(
|
||||
cd "${REPO_ROOT}"
|
||||
"${REPO_ROOT}/target/debug/lesavka-sync-analyze" \
|
||||
--event-width-codes "${PROBE_EVENT_WIDTH_CODES}" \
|
||||
--report-dir "${ANALYSIS_DIR}" \
|
||||
"${OBSERVER_CAPTURE_LOCAL}"
|
||||
) || status=$?
|
||||
|
||||
if (( status != 0 )); then
|
||||
echo " ↪ analyzer failed for observer recording; status=${status}"
|
||||
if [[ "${LESAVKA_MEET_ANALYSIS_REQUIRED}" == "1" ]]; then
|
||||
exit "${status}"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
echo " ↪ observer_report=${ANALYSIS_DIR}/report.txt"
|
||||
}
|
||||
|
||||
maybe_open_local_review() {
|
||||
if [[ "${LESAVKA_MEET_LOCAL_REVIEW}" != "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if [[ -n "${OBSERVER_CAPTURE_LOCAL}" ]]; then
|
||||
xdg-open "${OBSERVER_CAPTURE_LOCAL}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ -f "${ANALYSIS_DIR}/report.txt" ]]; then
|
||||
xdg-open "${ANALYSIS_DIR}/report.txt" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
validate_start_delay
|
||||
write_operator_checklist
|
||||
|
||||
echo "==> Google Meet observer probe"
|
||||
echo " ↪ server_addr=${LESAVKA_SERVER_ADDR}"
|
||||
echo " ↪ meet_url=${LESAVKA_GOOGLE_MEET_URL:-manual}"
|
||||
echo " ↪ artifact_dir=${ARTIFACT_DIR}"
|
||||
echo " ↪ run_log=${RUN_LOG}"
|
||||
echo " ↪ no Google credentials, sudo, or browser automation are used"
|
||||
|
||||
build_probe_tools
|
||||
print_versions || true
|
||||
maybe_open_meet_url
|
||||
|
||||
if (( LESAVKA_MEET_START_DELAY_SECONDS > 0 )); then
|
||||
echo "==> start delay: ${LESAVKA_MEET_START_DELAY_SECONDS}s"
|
||||
sleep "${LESAVKA_MEET_START_DELAY_SECONDS}"
|
||||
fi
|
||||
|
||||
echo "==> operator checklist written"
|
||||
sed 's/^/ | /' "${OPERATOR_CHECKLIST}"
|
||||
prompt_or_sleep "Join Meet on RCT, join as observer here, pin the RCT tile, and start observer recording" 30
|
||||
record_operator_ready_time
|
||||
run_synthetic_probe
|
||||
prompt_or_sleep "Stop/save the observer recording now" 5
|
||||
prompt_for_observer_capture_path
|
||||
copy_observer_capture_if_present
|
||||
analyze_observer_capture
|
||||
maybe_open_local_review
|
||||
|
||||
echo "==> done"
|
||||
echo "artifact_dir: ${ARTIFACT_DIR}"
|
||||
echo "operator_checklist: ${OPERATOR_CHECKLIST}"
|
||||
echo "manual_timing_json: ${MANUAL_TIMING_JSON}"
|
||||
echo "client_timeline_json: ${CLIENT_TIMELINE_JSON}"
|
||||
[[ -n "${OBSERVER_CAPTURE_LOCAL}" ]] && echo "observer_capture: ${OBSERVER_CAPTURE_LOCAL}"
|
||||
[[ -f "${ANALYSIS_DIR}/report.txt" ]] && echo "observer_report: ${ANALYSIS_DIR}/report.txt"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@ -10,7 +10,7 @@ bench = false
|
||||
|
||||
[package]
|
||||
name = "lesavka_server"
|
||||
version = "0.21.9"
|
||||
version = "0.21.10"
|
||||
edition = "2024"
|
||||
autobins = false
|
||||
|
||||
|
||||
@ -3,7 +3,8 @@
|
||||
//! Scope: statically guard the browser, mirrored, and local stimulus probe
|
||||
//! drivers used for client-to-server transport tuning.
|
||||
//! Targets: `scripts/manual/run_upstream_browser_av_sync.sh`,
|
||||
//! `scripts/manual/run_upstream_mirrored_av_sync.sh`, and local stimulus code.
|
||||
//! `scripts/manual/run_upstream_mirrored_av_sync.sh`,
|
||||
//! `scripts/manual/run_google_meet_observer_probe.sh`, and local stimulus code.
|
||||
//! Why: after server-to-RCT calibration, these scripts become the next tuning
|
||||
//! surface and must not drift back to synthetic or split upstream paths.
|
||||
|
||||
@ -11,6 +12,8 @@ const BROWSER_SYNC_SCRIPT: &str =
|
||||
include_str!("../../scripts/manual/run_upstream_browser_av_sync.sh");
|
||||
const MIRRORED_SYNC_SCRIPT: &str =
|
||||
include_str!("../../scripts/manual/run_upstream_mirrored_av_sync.sh");
|
||||
const GOOGLE_MEET_OBSERVER_SCRIPT: &str =
|
||||
include_str!("../../scripts/manual/run_google_meet_observer_probe.sh");
|
||||
const LOCAL_STIMULUS: &str = include_str!("../../scripts/manual/local_av_stimulus.py");
|
||||
const SYNC_PROBE_RUNNER: &str = include_str!("../../client/src/sync_probe/runner.rs");
|
||||
const SYNC_PROBE_BUNDLED_TRANSPORT: &str =
|
||||
@ -208,6 +211,52 @@ fn mirrored_sync_script_uses_real_client_capture_path() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn google_meet_observer_probe_is_manual_and_uses_bundled_synthetic_media() {
|
||||
for expected in [
|
||||
"Google Meet observer probe checklist",
|
||||
"no Google credentials, sudo, or browser automation are used",
|
||||
"LESAVKA_GOOGLE_MEET_URL=${LESAVKA_GOOGLE_MEET_URL:-}",
|
||||
"LESAVKA_MEET_OBSERVER_CAPTURE=${LESAVKA_MEET_OBSERVER_CAPTURE:-}",
|
||||
"LESAVKA_MEET_SKIP_PROMPTS=${LESAVKA_MEET_SKIP_PROMPTS:-0}",
|
||||
"LESAVKA_MEET_START_DELAY_SECONDS=${LESAVKA_MEET_START_DELAY_SECONDS:-0}",
|
||||
"LESAVKA_MEET_START_DELAY_SECONDS must be a non-negative integer",
|
||||
"LESAVKA_MEET_ANALYSIS_REQUIRED=${LESAVKA_MEET_ANALYSIS_REQUIRED:-0}",
|
||||
"lesavka-sync-probe",
|
||||
"--event-width-codes \"${PROBE_EVENT_WIDTH_CODES}\"",
|
||||
"--timeline-json \"${CLIENT_TIMELINE_JSON}\"",
|
||||
"Join Meet on RCT, join as observer here, pin the RCT tile",
|
||||
"Stop/save the observer recording now",
|
||||
"Path to observer recording for analysis (blank to skip)",
|
||||
"lesavka-sync-analyze",
|
||||
"--report-dir \"${ANALYSIS_DIR}\"",
|
||||
"manual-timing.json",
|
||||
"observer-google-meet-capture",
|
||||
"operator-checklist.txt",
|
||||
"Observer capture times are operator-confirmed",
|
||||
] {
|
||||
assert!(
|
||||
GOOGLE_MEET_OBSERVER_SCRIPT.contains(expected),
|
||||
"Google Meet observer probe should contain {expected}"
|
||||
);
|
||||
}
|
||||
for forbidden in [
|
||||
"google-chrome --remote-debugging",
|
||||
"chromedriver",
|
||||
"geckodriver",
|
||||
"sudo ",
|
||||
"PASSWORD",
|
||||
"VAULT",
|
||||
"vault",
|
||||
"read -s",
|
||||
] {
|
||||
assert!(
|
||||
!GOOGLE_MEET_OBSERVER_SCRIPT.contains(forbidden),
|
||||
"Google Meet observer probe must stay manual/non-secret: {forbidden}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_stimulus_matches_sync_analyzer_pulse_contract() {
|
||||
for expected in [
|
||||
@ -251,3 +300,21 @@ fn manual_probe_python_servers_use_reentrant_state_locks() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_browser_probe_scripts_stay_under_loc_guard() {
|
||||
for (name, contents) in [
|
||||
("run_upstream_browser_av_sync.sh", BROWSER_SYNC_SCRIPT),
|
||||
(
|
||||
"run_google_meet_observer_probe.sh",
|
||||
GOOGLE_MEET_OBSERVER_SCRIPT,
|
||||
),
|
||||
("local_av_stimulus.py", LOCAL_STIMULUS),
|
||||
] {
|
||||
let loc = contents.lines().count();
|
||||
assert!(
|
||||
loc <= 500,
|
||||
"{name} should stay under the 500 LOC refactor guard; saw {loc}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user