510 lines
20 KiB
Bash
Executable File
510 lines
20 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# scripts/manual/run_server_to_rc_mode_matrix.sh
|
|
# Manual: validate server-generated UVC/UAC output against the RC target across
|
|
# the UVC modes the UI advertises. This is still a hardware-in-the-loop probe:
|
|
# it captures the real Tethys UVC/UAC endpoints and summarizes sync,
|
|
# freshness, and smoothness for each mode.
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
|
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"
|
|
|
|
TETHYS_HOST=${TETHYS_HOST:-tethys}
|
|
LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST:-theia}
|
|
LESAVKA_SERVER_CONNECT_HOST=${LESAVKA_SERVER_CONNECT_HOST:-38.28.125.112}
|
|
LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-auto}
|
|
LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}
|
|
LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}
|
|
LESAVKA_SERVER_REPO=${LESAVKA_SERVER_REPO:-/home/brad/Development/lesavka}
|
|
SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=30"}
|
|
|
|
LESAVKA_SERVER_RC_MODES=${LESAVKA_SERVER_RC_MODES:-640x480@20,1280x720@30,1920x1080@20,1920x1080@30}
|
|
LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US:-170000}
|
|
LESAVKA_SERVER_RC_MODE_DELAYS_US=${LESAVKA_SERVER_RC_MODE_DELAYS_US:-640x480@20=170000,1280x720@30=170000,1920x1080@20=170000,1920x1080@30=170000}
|
|
LESAVKA_SERVER_RC_RECONFIGURE=${LESAVKA_SERVER_RC_RECONFIGURE:-0}
|
|
LESAVKA_SERVER_RC_RECONFIGURE_REF=${LESAVKA_SERVER_RC_RECONFIGURE_REF:-master}
|
|
LESAVKA_SERVER_RC_RECONFIGURE_COMMAND=${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND:-}
|
|
LESAVKA_SERVER_RC_CONTINUE_ON_FAIL=${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL:-1}
|
|
|
|
LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350}
|
|
LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS:-100}
|
|
LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}
|
|
LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS:-1}
|
|
LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS=${LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS:-1}
|
|
|
|
LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0}
|
|
LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0}
|
|
LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS=${LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS:-1}
|
|
LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS=${LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS:-1}
|
|
LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES:-12}
|
|
LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES:-12}
|
|
LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES:-5}
|
|
LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS=${LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS:-2}
|
|
|
|
PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-20}
|
|
PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4}
|
|
PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,1,3,2,4,1,1,3,1,4,2,1,2,3,4,1,3,2,2,4,1,2,4,3,1,1,4,2,3,1,2}
|
|
REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}
|
|
REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-cfr}
|
|
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse}
|
|
REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto}
|
|
LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}
|
|
|
|
STAMP="$(date +%Y%m%d-%H%M%S)"
|
|
LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}
|
|
MATRIX_REPORT_DIR=${MATRIX_REPORT_DIR:-"${LOCAL_OUTPUT_DIR%/}/lesavka-server-rc-mode-matrix-${STAMP}"}
|
|
MATRIX_SUMMARY_JSON="${MATRIX_REPORT_DIR}/mode-matrix-summary.json"
|
|
MATRIX_SUMMARY_CSV="${MATRIX_REPORT_DIR}/mode-matrix-summary.csv"
|
|
MATRIX_SUMMARY_TXT="${MATRIX_REPORT_DIR}/mode-matrix-summary.txt"
|
|
mkdir -p "${MATRIX_REPORT_DIR}"
|
|
|
|
mode_id() {
|
|
local mode=$1
|
|
printf '%s\n' "${mode//@/_}" | tr -c '[:alnum:]_.-' '_'
|
|
}
|
|
|
|
parse_mode() {
|
|
local mode=$1
|
|
if [[ ! "${mode}" =~ ^([0-9]+)x([0-9]+)@([0-9]+)$ ]]; then
|
|
printf 'invalid mode %s; expected WIDTHxHEIGHT@FPS\n' "${mode}" >&2
|
|
exit 64
|
|
fi
|
|
printf '%s %s %s\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}"
|
|
}
|
|
|
|
lookup_video_delay_us() {
|
|
local mode=$1
|
|
local entry key value
|
|
IFS=',' read -r -a delay_entries <<<"${LESAVKA_SERVER_RC_MODE_DELAYS_US}"
|
|
for entry in "${delay_entries[@]}"; do
|
|
key=${entry%%=*}
|
|
value=${entry#*=}
|
|
if [[ "${key}" == "${mode}" && "${value}" =~ ^-?[0-9]+$ ]]; then
|
|
printf '%s\n' "${value}"
|
|
return 0
|
|
fi
|
|
done
|
|
printf '%s\n' "${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US}"
|
|
}
|
|
|
|
reconfigure_server_mode() {
|
|
local mode=$1
|
|
local width=$2
|
|
local height=$3
|
|
local fps=$4
|
|
[[ "${LESAVKA_SERVER_RC_RECONFIGURE}" != "0" ]] || return 0
|
|
|
|
echo "==> reconfiguring ${LESAVKA_SERVER_HOST} UVC gadget for ${mode}"
|
|
if [[ -n "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}" ]]; then
|
|
LESAVKA_MODE="${mode}" \
|
|
LESAVKA_UVC_WIDTH="${width}" \
|
|
LESAVKA_UVC_HEIGHT="${height}" \
|
|
LESAVKA_UVC_FPS="${fps}" \
|
|
bash -c "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}"
|
|
return 0
|
|
fi
|
|
|
|
ssh ${SSH_OPTS} "${LESAVKA_SERVER_HOST}" bash -s -- \
|
|
"${LESAVKA_SERVER_REPO}" \
|
|
"${LESAVKA_SERVER_RC_RECONFIGURE_REF}" \
|
|
"${width}" \
|
|
"${height}" \
|
|
"${fps}" <<'REMOTE_RECONFIGURE'
|
|
set -euo pipefail
|
|
repo=$1
|
|
ref=$2
|
|
width=$3
|
|
height=$4
|
|
fps=$5
|
|
cd "${repo}"
|
|
git fetch --all --prune
|
|
git checkout "${ref}"
|
|
git pull --ff-only
|
|
sudo env \
|
|
LESAVKA_REF="${ref}" \
|
|
LESAVKA_INSTALL_UVC_CODEC=mjpeg \
|
|
LESAVKA_UVC_WIDTH="${width}" \
|
|
LESAVKA_UVC_HEIGHT="${height}" \
|
|
LESAVKA_UVC_FPS="${fps}" \
|
|
LESAVKA_UVC_INTERVAL="$((10000000 / fps))" \
|
|
./scripts/install/server.sh
|
|
REMOTE_RECONFIGURE
|
|
}
|
|
|
|
write_mode_result() {
|
|
local mode=$1
|
|
local width=$2
|
|
local height=$3
|
|
local fps=$4
|
|
local video_delay_us=$5
|
|
local run_status=$6
|
|
local run_log=$7
|
|
local artifact_dir=$8
|
|
local output_json=$9
|
|
|
|
python3 - <<'PY' \
|
|
"${mode}" \
|
|
"${width}" \
|
|
"${height}" \
|
|
"${fps}" \
|
|
"${video_delay_us}" \
|
|
"${run_status}" \
|
|
"${run_log}" \
|
|
"${artifact_dir}" \
|
|
"${output_json}" \
|
|
"${LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS}" \
|
|
"${LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS}" \
|
|
"${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS}" \
|
|
"${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS}" \
|
|
"${LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS}" \
|
|
"${LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS}" \
|
|
"${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES}" \
|
|
"${LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES}" \
|
|
"${LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES}" \
|
|
"${LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS}"
|
|
import json
|
|
import math
|
|
import pathlib
|
|
import sys
|
|
|
|
(
|
|
mode,
|
|
width_raw,
|
|
height_raw,
|
|
fps_raw,
|
|
video_delay_raw,
|
|
run_status_raw,
|
|
run_log,
|
|
artifact_dir_raw,
|
|
output_json,
|
|
require_sync_raw,
|
|
require_freshness_raw,
|
|
max_video_hiccups_raw,
|
|
max_audio_hiccups_raw,
|
|
max_video_jitter_raw,
|
|
max_audio_jitter_raw,
|
|
max_missing_raw,
|
|
max_undecodable_raw,
|
|
max_duplicates_raw,
|
|
max_low_rms_raw,
|
|
) = sys.argv[1:]
|
|
|
|
|
|
def as_bool(value):
|
|
return str(value).strip().lower() not in {"", "0", "false", "no", "off"}
|
|
|
|
|
|
def as_int(value, default=0):
|
|
try:
|
|
return int(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def as_float(value, default=0.0):
|
|
try:
|
|
result = float(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
return result if math.isfinite(result) else default
|
|
|
|
|
|
def load_json(path):
|
|
try:
|
|
return json.loads(pathlib.Path(path).read_text())
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def nested(mapping, *keys, default=None):
|
|
value = mapping
|
|
for key in keys:
|
|
if not isinstance(value, dict):
|
|
return default
|
|
value = value.get(key)
|
|
return default if value is None else value
|
|
|
|
|
|
artifact_dir = pathlib.Path(artifact_dir_raw) if artifact_dir_raw else pathlib.Path()
|
|
report = load_json(artifact_dir / "report.json")
|
|
correlation = load_json(artifact_dir / "output-delay-correlation.json")
|
|
freshness = correlation.get("freshness") or {}
|
|
smoothness = correlation.get("smoothness") or {}
|
|
video = smoothness.get("video") or {}
|
|
audio = smoothness.get("audio") or {}
|
|
audio_cadence = audio.get("packet_cadence") or {}
|
|
audio_rms = audio.get("rms_continuity") or {}
|
|
verdict = report.get("verdict") or {}
|
|
|
|
sync_pass = verdict.get("passed") is True
|
|
freshness_status = freshness.get("status", "unknown")
|
|
freshness_pass = freshness_status == "pass"
|
|
run_status = as_int(run_status_raw, 1)
|
|
|
|
video_hiccups = as_int(video.get("hiccup_count"), 0)
|
|
audio_hiccups = as_int(audio_cadence.get("hiccup_count"), 0)
|
|
video_jitter = as_float(nested(video, "jitter_stats", "p95_jitter_ms"), 0.0)
|
|
audio_jitter = as_float(nested(audio_cadence, "jitter_stats", "p95_jitter_ms"), 0.0)
|
|
missing = as_int(video.get("estimated_missing_frames"), 0)
|
|
undecodable = as_int(video.get("undecodable_frames"), 0)
|
|
duplicates = as_int(video.get("duplicate_frames"), 0)
|
|
low_rms = as_int(audio_rms.get("low_rms_window_count"), 0)
|
|
|
|
reasons = []
|
|
if run_status != 0:
|
|
reasons.append(f"probe command exited {run_status}")
|
|
if as_bool(require_sync_raw) and not sync_pass:
|
|
reasons.append(f"sync did not pass: {verdict.get('status', 'unknown')}")
|
|
if as_bool(require_freshness_raw) and not freshness_pass:
|
|
reasons.append(f"freshness did not pass: {freshness_status}")
|
|
if video_hiccups > as_int(max_video_hiccups_raw):
|
|
reasons.append(f"video hiccups {video_hiccups} > {max_video_hiccups_raw}")
|
|
if audio_hiccups > as_int(max_audio_hiccups_raw):
|
|
reasons.append(f"audio hiccups {audio_hiccups} > {max_audio_hiccups_raw}")
|
|
if video_jitter > as_float(max_video_jitter_raw):
|
|
reasons.append(f"video p95 jitter {video_jitter:.1f}ms > {as_float(max_video_jitter_raw):.1f}ms")
|
|
if audio_jitter > as_float(max_audio_jitter_raw):
|
|
reasons.append(f"audio p95 jitter {audio_jitter:.1f}ms > {as_float(max_audio_jitter_raw):.1f}ms")
|
|
if missing > as_int(max_missing_raw):
|
|
reasons.append(f"estimated missing video frames {missing} > {max_missing_raw}")
|
|
if undecodable > as_int(max_undecodable_raw):
|
|
reasons.append(f"undecodable video frames {undecodable} > {max_undecodable_raw}")
|
|
if duplicates > as_int(max_duplicates_raw):
|
|
reasons.append(f"duplicate video frames {duplicates} > {max_duplicates_raw}")
|
|
if low_rms > as_int(max_low_rms_raw):
|
|
reasons.append(f"low-RMS audio windows {low_rms} > {max_low_rms_raw}")
|
|
|
|
artifact = {
|
|
"schema": "lesavka.server-rc-mode-result.v1",
|
|
"mode": mode,
|
|
"width": as_int(width_raw),
|
|
"height": as_int(height_raw),
|
|
"fps": as_int(fps_raw),
|
|
"audio_delay_us": 0,
|
|
"video_delay_us": as_int(video_delay_raw),
|
|
"run_status": run_status,
|
|
"run_log": run_log,
|
|
"artifact_dir": str(artifact_dir),
|
|
"report_json": str(artifact_dir / "report.json"),
|
|
"correlation_json": str(artifact_dir / "output-delay-correlation.json"),
|
|
"passed": not reasons,
|
|
"failure_reasons": reasons,
|
|
"sync": {
|
|
"passed": sync_pass,
|
|
"status": verdict.get("status", "unknown"),
|
|
"reason": verdict.get("reason", ""),
|
|
"p95_abs_skew_ms": as_float(verdict.get("p95_abs_skew_ms"), 0.0),
|
|
"median_skew_ms": as_float(report.get("median_skew_ms"), 0.0),
|
|
"drift_ms": as_float(report.get("drift_ms"), 0.0),
|
|
"paired_event_count": as_int(report.get("paired_event_count"), 0),
|
|
},
|
|
"freshness": {
|
|
"status": freshness_status,
|
|
"reason": freshness.get("reason", ""),
|
|
"worst_event_age_p95_ms": freshness.get("worst_event_age_p95_ms"),
|
|
"clock_uncertainty_ms": freshness.get("clock_uncertainty_ms"),
|
|
"worst_event_age_with_uncertainty_ms": freshness.get("worst_event_age_with_uncertainty_ms"),
|
|
"worst_freshness_drift_ms": freshness.get("worst_freshness_drift_ms"),
|
|
"max_age_limit_ms": freshness.get("max_age_limit_ms"),
|
|
},
|
|
"smoothness": {
|
|
"video_frames": as_int(video.get("timestamps"), 0),
|
|
"video_p95_jitter_ms": video_jitter,
|
|
"video_max_interval_ms": as_float(nested(video, "interval_stats", "max_interval_ms"), 0.0),
|
|
"video_hiccups": video_hiccups,
|
|
"video_decoded_frames": as_int(video.get("decoded_frames"), 0),
|
|
"video_duplicate_frames": duplicates,
|
|
"video_estimated_missing_frames": missing,
|
|
"video_undecodable_frames": undecodable,
|
|
"audio_packets": as_int(audio_cadence.get("timestamps"), 0),
|
|
"audio_p95_jitter_ms": audio_jitter,
|
|
"audio_max_interval_ms": as_float(nested(audio_cadence, "interval_stats", "max_interval_ms"), 0.0),
|
|
"audio_hiccups": audio_hiccups,
|
|
"audio_low_rms_windows": low_rms,
|
|
"audio_median_rms": as_float(nested(audio_rms, "rms_stats", "median_rms"), 0.0),
|
|
},
|
|
}
|
|
pathlib.Path(output_json).write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n")
|
|
PY
|
|
}
|
|
|
|
summarize_matrix() {
|
|
python3 - <<'PY' "${MATRIX_REPORT_DIR}" "${MATRIX_SUMMARY_JSON}" "${MATRIX_SUMMARY_CSV}" "${MATRIX_SUMMARY_TXT}"
|
|
import csv
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
root = pathlib.Path(sys.argv[1])
|
|
summary_json = pathlib.Path(sys.argv[2])
|
|
summary_csv = pathlib.Path(sys.argv[3])
|
|
summary_txt = pathlib.Path(sys.argv[4])
|
|
results = []
|
|
for path in sorted(root.glob("*/mode-result.json")):
|
|
try:
|
|
results.append(json.loads(path.read_text()))
|
|
except Exception:
|
|
continue
|
|
|
|
summary = {
|
|
"schema": "lesavka.server-rc-mode-matrix-summary.v1",
|
|
"artifact_dir": str(root),
|
|
"passed": bool(results) and all(result.get("passed") for result in results),
|
|
"mode_count": len(results),
|
|
"results": results,
|
|
}
|
|
summary_json.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
|
|
|
|
fieldnames = [
|
|
"mode",
|
|
"passed",
|
|
"video_delay_us",
|
|
"sync_status",
|
|
"p95_abs_skew_ms",
|
|
"median_skew_ms",
|
|
"sync_drift_ms",
|
|
"freshness_status",
|
|
"freshness_budget_ms",
|
|
"freshness_drift_ms",
|
|
"video_hiccups",
|
|
"video_missing",
|
|
"video_undecodable",
|
|
"audio_hiccups",
|
|
"audio_low_rms",
|
|
"artifact_dir",
|
|
]
|
|
with summary_csv.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
for result in results:
|
|
writer.writerow({
|
|
"mode": result.get("mode"),
|
|
"passed": result.get("passed"),
|
|
"video_delay_us": result.get("video_delay_us"),
|
|
"sync_status": (result.get("sync") or {}).get("status"),
|
|
"p95_abs_skew_ms": (result.get("sync") or {}).get("p95_abs_skew_ms"),
|
|
"median_skew_ms": (result.get("sync") or {}).get("median_skew_ms"),
|
|
"sync_drift_ms": (result.get("sync") or {}).get("drift_ms"),
|
|
"freshness_status": (result.get("freshness") or {}).get("status"),
|
|
"freshness_budget_ms": (result.get("freshness") or {}).get("worst_event_age_with_uncertainty_ms"),
|
|
"freshness_drift_ms": (result.get("freshness") or {}).get("worst_freshness_drift_ms"),
|
|
"video_hiccups": (result.get("smoothness") or {}).get("video_hiccups"),
|
|
"video_missing": (result.get("smoothness") or {}).get("video_estimated_missing_frames"),
|
|
"video_undecodable": (result.get("smoothness") or {}).get("video_undecodable_frames"),
|
|
"audio_hiccups": (result.get("smoothness") or {}).get("audio_hiccups"),
|
|
"audio_low_rms": (result.get("smoothness") or {}).get("audio_low_rms_windows"),
|
|
"artifact_dir": result.get("artifact_dir"),
|
|
})
|
|
|
|
lines = [
|
|
f"Server-to-RC mode matrix for {root}",
|
|
f"- modes: {len(results)}",
|
|
f"- verdict: {'pass' if summary['passed'] else 'fail'}",
|
|
]
|
|
for result in results:
|
|
sync = result.get("sync") or {}
|
|
freshness = result.get("freshness") or {}
|
|
smooth = result.get("smoothness") or {}
|
|
marker = "PASS" if result.get("passed") else "FAIL"
|
|
lines.append(
|
|
f"- {marker} {result.get('mode')}: "
|
|
f"sync {sync.get('status')} p95={sync.get('p95_abs_skew_ms', 0.0):.1f}ms median={sync.get('median_skew_ms', 0.0):+.1f}ms; "
|
|
f"freshness {freshness.get('status')} budget={freshness.get('worst_event_age_with_uncertainty_ms') or 0.0:.1f}ms; "
|
|
f"smooth video hiccups={smooth.get('video_hiccups', 0)} missing={smooth.get('video_estimated_missing_frames', 0)} undecodable={smooth.get('video_undecodable_frames', 0)} audio hiccups={smooth.get('audio_hiccups', 0)}"
|
|
)
|
|
for reason in result.get("failure_reasons") or []:
|
|
lines.append(f" reason: {reason}")
|
|
summary_txt.write_text("\n".join(lines) + "\n")
|
|
print("\n".join(lines))
|
|
PY
|
|
}
|
|
|
|
echo "==> server-to-RC mode matrix"
|
|
echo " ↪ modes=${LESAVKA_SERVER_RC_MODES}"
|
|
echo " ↪ delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}"
|
|
echo " ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}"
|
|
echo " ↪ artifact_dir=${MATRIX_REPORT_DIR}"
|
|
|
|
IFS=',' read -r -a modes <<<"${LESAVKA_SERVER_RC_MODES}"
|
|
for mode in "${modes[@]}"; do
|
|
mode="${mode//[[:space:]]/}"
|
|
[[ -n "${mode}" ]] || continue
|
|
read -r width height fps < <(parse_mode "${mode}")
|
|
video_delay_us="$(lookup_video_delay_us "${mode}")"
|
|
id="$(mode_id "${mode}")"
|
|
mode_dir="${MATRIX_REPORT_DIR}/${id}"
|
|
mode_log="${mode_dir}/mode-run.log"
|
|
mode_result="${mode_dir}/mode-result.json"
|
|
mkdir -p "${mode_dir}"
|
|
|
|
echo "==> mode ${mode}: video_delay_us=${video_delay_us}"
|
|
reconfigure_server_mode "${mode}" "${width}" "${height}" "${fps}"
|
|
|
|
set +e
|
|
TETHYS_HOST="${TETHYS_HOST}" \
|
|
LESAVKA_SERVER_HOST="${LESAVKA_SERVER_HOST}" \
|
|
LESAVKA_SERVER_CONNECT_HOST="${LESAVKA_SERVER_CONNECT_HOST}" \
|
|
LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_ADDR}" \
|
|
LESAVKA_SERVER_SCHEME="${LESAVKA_SERVER_SCHEME}" \
|
|
LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \
|
|
SSH_OPTS="${SSH_OPTS}" \
|
|
REMOTE_PULSE_CAPTURE_TOOL="${REMOTE_PULSE_CAPTURE_TOOL}" \
|
|
REMOTE_PULSE_VIDEO_MODE="${REMOTE_PULSE_VIDEO_MODE}" \
|
|
REMOTE_CAPTURE_STACK="${REMOTE_CAPTURE_STACK}" \
|
|
REMOTE_AUDIO_SOURCE="${REMOTE_AUDIO_SOURCE}" \
|
|
VIDEO_SIZE="${width}x${height}" \
|
|
VIDEO_FPS="${fps}" \
|
|
VIDEO_FORMAT=mjpeg \
|
|
REMOTE_EXPECT_UVC_WIDTH="${width}" \
|
|
REMOTE_EXPECT_UVC_HEIGHT="${height}" \
|
|
REMOTE_EXPECT_UVC_FPS="${fps}" \
|
|
LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US="${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US}" \
|
|
LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US="${video_delay_us}" \
|
|
LESAVKA_OUTPUT_DELAY_APPLY=0 \
|
|
LESAVKA_OUTPUT_DELAY_SAVE=0 \
|
|
LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0 \
|
|
LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" \
|
|
LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}" \
|
|
LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}" \
|
|
PROBE_EVENT_WIDTH_CODES="${PROBE_EVENT_WIDTH_CODES}" \
|
|
PROBE_DURATION_SECONDS="${PROBE_DURATION_SECONDS}" \
|
|
PROBE_WARMUP_SECONDS="${PROBE_WARMUP_SECONDS}" \
|
|
LOCAL_OUTPUT_DIR="${mode_dir}" \
|
|
"${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${mode_log}"
|
|
run_status=${PIPESTATUS[0]}
|
|
set -e
|
|
|
|
artifact_dir="$(awk -F': ' '/^artifact_dir: / {print $2}' "${mode_log}" | tail -n1)"
|
|
if [[ -z "${artifact_dir}" ]]; then
|
|
artifact_dir="$(find "${mode_dir}" -mindepth 1 -maxdepth 1 -type d -name 'lesavka-output-delay-probe-*' | sort | tail -n1 || true)"
|
|
fi
|
|
write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${video_delay_us}" "${run_status}" "${mode_log}" "${artifact_dir}" "${mode_result}"
|
|
|
|
if [[ "${run_status}" -ne 0 && "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then
|
|
break
|
|
fi
|
|
done
|
|
|
|
summarize_matrix
|
|
|
|
echo "==> done"
|
|
echo "artifact_dir: ${MATRIX_REPORT_DIR}"
|
|
echo "mode_matrix_summary_json: ${MATRIX_SUMMARY_JSON}"
|
|
echo "mode_matrix_summary_csv: ${MATRIX_SUMMARY_CSV}"
|
|
echo "mode_matrix_summary_txt: ${MATRIX_SUMMARY_TXT}"
|
|
|
|
if python3 - <<'PY' "${MATRIX_SUMMARY_JSON}"
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
summary = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
raise SystemExit(0 if summary.get("passed") else 1)
|
|
PY
|
|
then
|
|
exit 0
|
|
fi
|
|
|
|
exit 95
|