test(server-rc): retry signal readiness with layer evidence
This commit is contained in:
parent
a707331cda
commit
99656b05d2
6
Cargo.lock
generated
6
Cargo.lock
generated
@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
||||
|
||||
[[package]]
|
||||
name = "lesavka_client"
|
||||
version = "0.19.25"
|
||||
version = "0.19.26"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-stream",
|
||||
@ -1686,7 +1686,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lesavka_common"
|
||||
version = "0.19.25"
|
||||
version = "0.19.26"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
@ -1698,7 +1698,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lesavka_server"
|
||||
version = "0.19.25"
|
||||
version = "0.19.26"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
|
||||
@ -4,7 +4,7 @@ path = "src/main.rs"
|
||||
|
||||
[package]
|
||||
name = "lesavka_client"
|
||||
version = "0.19.25"
|
||||
version = "0.19.26"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lesavka_common"
|
||||
version = "0.19.25"
|
||||
version = "0.19.26"
|
||||
edition = "2024"
|
||||
build = "build.rs"
|
||||
|
||||
|
||||
@ -78,8 +78,10 @@ LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_
|
||||
LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS:-0}
|
||||
LESAVKA_SERVER_RC_SIGNAL_READY=${LESAVKA_SERVER_RC_SIGNAL_READY:-1}
|
||||
LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS:-3}
|
||||
LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS:-8}
|
||||
LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS:-12}
|
||||
LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS:-1}
|
||||
LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS:-4}
|
||||
LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS:-5}
|
||||
|
||||
LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0}
|
||||
LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0}
|
||||
@ -364,6 +366,229 @@ raise SystemExit(0 if passed else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
write_signal_readiness_attempt_result() {
|
||||
local attempt=$1
|
||||
local artifact_dir=$2
|
||||
local run_status=$3
|
||||
local run_log=$4
|
||||
local min_pairs=$5
|
||||
local output_json=$6
|
||||
python3 - <<'PY' \
|
||||
"${attempt}" \
|
||||
"${artifact_dir}" \
|
||||
"${run_status}" \
|
||||
"${run_log}" \
|
||||
"${min_pairs}" \
|
||||
"${output_json}"
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
(
|
||||
attempt_raw,
|
||||
artifact_dir_raw,
|
||||
run_status_raw,
|
||||
run_log_raw,
|
||||
min_pairs_raw,
|
||||
output_json_raw,
|
||||
) = sys.argv[1:]
|
||||
|
||||
def as_int(value, default=0):
|
||||
try:
|
||||
return int(str(value).strip())
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def as_float(value, default=0.0):
|
||||
try:
|
||||
return float(str(value).strip())
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def load_json(path):
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def read_text(path):
|
||||
try:
|
||||
return path.read_text(errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
attempt = as_int(attempt_raw)
|
||||
run_status = as_int(run_status_raw)
|
||||
min_pairs = as_int(min_pairs_raw, 1)
|
||||
run_log = pathlib.Path(run_log_raw)
|
||||
artifact_dir = pathlib.Path(artifact_dir_raw) if artifact_dir_raw else None
|
||||
if artifact_dir is None or not artifact_dir.exists():
|
||||
match = re.findall(r"^artifact_dir:\s*(.+)$", read_text(run_log), flags=re.MULTILINE)
|
||||
if match:
|
||||
artifact_dir = pathlib.Path(match[-1].strip())
|
||||
if artifact_dir is None:
|
||||
artifact_dir = pathlib.Path("__missing_signal_readiness_artifact__")
|
||||
|
||||
report = load_json(artifact_dir / "report.json")
|
||||
timeline = load_json(artifact_dir / "server-output-timeline.json")
|
||||
error_log = artifact_dir / "analysis-error.log"
|
||||
error_text = read_text(error_log)
|
||||
if not error_text:
|
||||
error_text = read_text(run_log)
|
||||
|
||||
events = timeline.get("events") or []
|
||||
server_event_count = len(events)
|
||||
server_video_handoff_count = sum(
|
||||
1
|
||||
for event in events
|
||||
if event.get("video_feed_unix_ns") is not None
|
||||
or event.get("video_feed_monotonic_us") is not None
|
||||
)
|
||||
server_audio_handoff_count = sum(
|
||||
1
|
||||
for event in events
|
||||
if event.get("audio_push_unix_ns") is not None
|
||||
or event.get("audio_push_monotonic_us") is not None
|
||||
)
|
||||
|
||||
verdict = report.get("verdict") or {}
|
||||
signature_coverage = report.get("signature_coverage") or {}
|
||||
video_events = as_int(report.get("video_event_count"))
|
||||
audio_events = as_int(report.get("audio_event_count"))
|
||||
paired_events = as_int(report.get("paired_event_count"))
|
||||
coded_pairs = as_int(signature_coverage.get("paired_event_count"), paired_events)
|
||||
expected_pairs = as_int(signature_coverage.get("expected_event_count"))
|
||||
raw_activity_delta_ms = as_float(report.get("activity_start_delta_ms"))
|
||||
analyzer_failure = ""
|
||||
|
||||
failure_match = re.search(
|
||||
r"coded pulse common window removed one stream entirely;[^\n]*"
|
||||
r"\(video=(?P<video>\d+)\s+audio=(?P<audio>\d+)\s+raw activity delta\s+"
|
||||
r"(?P<delta>[+-]?\d+(?:\.\d+)?)\s+ms\)",
|
||||
error_text,
|
||||
)
|
||||
if failure_match:
|
||||
video_events = max(video_events, as_int(failure_match.group("video")))
|
||||
audio_events = max(audio_events, as_int(failure_match.group("audio")))
|
||||
raw_activity_delta_ms = as_float(failure_match.group("delta"))
|
||||
analyzer_failure = failure_match.group(0)
|
||||
|
||||
sync_passed = verdict.get("passed") is True
|
||||
paired_ready = paired_events >= min_pairs or coded_pairs >= min_pairs
|
||||
video_ready = video_events >= min_pairs
|
||||
audio_ready = audio_events >= min_pairs
|
||||
passed = run_status == 0 and paired_ready and video_ready and audio_ready and sync_passed
|
||||
|
||||
reasons = []
|
||||
if run_status != 0:
|
||||
reasons.append(f"probe command exited {run_status}")
|
||||
if server_event_count == 0:
|
||||
reasons.append("server did not report generated coded events")
|
||||
if server_video_handoff_count == 0:
|
||||
reasons.append("server did not report video sink handoff events")
|
||||
if server_audio_handoff_count == 0:
|
||||
reasons.append("server did not report audio sink handoff events")
|
||||
if video_events < min_pairs:
|
||||
reasons.append(f"Tethys video coded events {video_events} < {min_pairs}")
|
||||
if audio_events < min_pairs:
|
||||
reasons.append(f"Tethys audio coded events {audio_events} < {min_pairs}")
|
||||
if max(paired_events, coded_pairs) < min_pairs:
|
||||
reasons.append(f"paired coded events {max(paired_events, coded_pairs)} < {min_pairs}")
|
||||
if report and not sync_passed:
|
||||
reasons.append(f"sync verdict did not pass: {verdict.get('status', 'unknown')}")
|
||||
if analyzer_failure:
|
||||
reasons.append(analyzer_failure)
|
||||
if not reasons and passed:
|
||||
reasons.append("ready")
|
||||
elif not reasons:
|
||||
reasons.append("signal readiness failed")
|
||||
|
||||
result = {
|
||||
"schema": "lesavka.server-rc-signal-readiness-attempt.v1",
|
||||
"attempt": attempt,
|
||||
"passed": passed,
|
||||
"reason": "; ".join(reasons),
|
||||
"run_status": run_status,
|
||||
"run_log": str(run_log),
|
||||
"artifact_dir": str(artifact_dir) if artifact_dir.exists() else "",
|
||||
"analysis_error_log": str(error_log) if error_log.exists() else "",
|
||||
"counts": {
|
||||
"server_events": server_event_count,
|
||||
"server_video_handoffs": server_video_handoff_count,
|
||||
"server_audio_handoffs": server_audio_handoff_count,
|
||||
"video_events": video_events,
|
||||
"audio_events": audio_events,
|
||||
"paired_events": paired_events,
|
||||
"coded_pairs": coded_pairs,
|
||||
"expected_pairs": expected_pairs,
|
||||
},
|
||||
"layers": {
|
||||
"server_generated_video": server_event_count > 0,
|
||||
"server_generated_audio": server_event_count > 0,
|
||||
"server_video_handoff": server_video_handoff_count > 0,
|
||||
"server_audio_handoff": server_audio_handoff_count > 0,
|
||||
"tethys_video_detected": video_events > 0,
|
||||
"tethys_audio_detected": audio_events > 0,
|
||||
"tethys_video_ready": video_ready,
|
||||
"tethys_audio_ready": audio_ready,
|
||||
"paired_coded_ready": paired_ready,
|
||||
"analyzer_sync_passed": sync_passed,
|
||||
},
|
||||
"sync": {
|
||||
"status": verdict.get("status", "unknown"),
|
||||
"passed": sync_passed,
|
||||
"p95_abs_skew_ms": as_float(verdict.get("p95_abs_skew_ms")),
|
||||
"median_skew_ms": as_float(report.get("median_skew_ms")),
|
||||
"drift_ms": as_float(report.get("drift_ms")),
|
||||
"raw_activity_delta_ms": raw_activity_delta_ms,
|
||||
},
|
||||
"analyzer_failure": analyzer_failure,
|
||||
}
|
||||
pathlib.Path(output_json_raw).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
||||
print(result["reason"])
|
||||
raise SystemExit(0 if passed else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
write_signal_readiness_attempts_summary() {
|
||||
local output_json=$1
|
||||
shift || true
|
||||
python3 - <<'PY' "${output_json}" "$@"
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
output = pathlib.Path(sys.argv[1])
|
||||
attempts = []
|
||||
for raw in sys.argv[2:]:
|
||||
path = pathlib.Path(raw)
|
||||
try:
|
||||
attempts.append(json.loads(path.read_text()))
|
||||
except Exception:
|
||||
attempts.append({
|
||||
"schema": "lesavka.server-rc-signal-readiness-attempt.v1",
|
||||
"attempt_json": str(path),
|
||||
"passed": False,
|
||||
"reason": "attempt result could not be read",
|
||||
})
|
||||
passed_attempt = next((attempt for attempt in attempts if attempt.get("passed")), None)
|
||||
summary = {
|
||||
"schema": "lesavka.server-rc-signal-readiness-summary.v1",
|
||||
"passed": passed_attempt is not None,
|
||||
"passed_attempt": passed_attempt.get("attempt") if passed_attempt else None,
|
||||
"attempt_count": len(attempts),
|
||||
"final_reason": (
|
||||
"ready"
|
||||
if passed_attempt
|
||||
else (attempts[-1].get("reason") if attempts else "no readiness attempts ran")
|
||||
),
|
||||
"attempts": attempts,
|
||||
}
|
||||
output.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
|
||||
PY
|
||||
}
|
||||
|
||||
write_signal_readiness_failure() {
|
||||
local mode=$1
|
||||
local width=$2
|
||||
@ -374,8 +599,9 @@ write_signal_readiness_failure() {
|
||||
local readiness_status=$7
|
||||
local readiness_log=$8
|
||||
local readiness_artifact_dir=$9
|
||||
local readiness_reason=${10}
|
||||
local output_json=${11}
|
||||
local readiness_attempts_json=${10}
|
||||
local readiness_reason=${11}
|
||||
local output_json=${12}
|
||||
python3 - <<'PY' \
|
||||
"${mode}" \
|
||||
"${width}" \
|
||||
@ -386,6 +612,7 @@ write_signal_readiness_failure() {
|
||||
"${readiness_status}" \
|
||||
"${readiness_log}" \
|
||||
"${readiness_artifact_dir}" \
|
||||
"${readiness_attempts_json}" \
|
||||
"${readiness_reason}" \
|
||||
"${output_json}"
|
||||
import json
|
||||
@ -402,6 +629,7 @@ import sys
|
||||
readiness_status_raw,
|
||||
readiness_log,
|
||||
readiness_artifact_dir,
|
||||
readiness_attempts_json,
|
||||
readiness_reason,
|
||||
output_json,
|
||||
) = sys.argv[1:]
|
||||
@ -421,6 +649,7 @@ def load_json(path):
|
||||
report = load_json(pathlib.Path(readiness_artifact_dir) / "report.json")
|
||||
verdict = report.get("verdict") or {}
|
||||
signature_coverage = report.get("signature_coverage") or {}
|
||||
readiness_attempts = load_json(readiness_attempts_json).get("attempts") if readiness_attempts_json else None
|
||||
result = {
|
||||
"schema": "lesavka.server-rc-mode-result.v1",
|
||||
"mode": mode,
|
||||
@ -440,6 +669,8 @@ result = {
|
||||
"reason": readiness_reason,
|
||||
"artifact_dir": readiness_artifact_dir,
|
||||
"run_log": readiness_log,
|
||||
"attempts_json": readiness_attempts_json,
|
||||
"attempts": readiness_attempts or [],
|
||||
},
|
||||
"sync": {
|
||||
"passed": verdict.get("passed") is True,
|
||||
@ -1516,6 +1747,28 @@ for result in results:
|
||||
f"valid={freshness.get('capture_timebase_valid')} "
|
||||
f"reason={freshness.get('capture_timebase_reason', '')}"
|
||||
)
|
||||
signal_readiness = result.get("signal_readiness") or {}
|
||||
signal_attempts = signal_readiness.get("attempts") or []
|
||||
if signal_readiness:
|
||||
lines.append(
|
||||
" signal readiness: "
|
||||
f"passed={signal_readiness.get('passed', False)} "
|
||||
f"attempts={len(signal_attempts)} "
|
||||
f"reason={signal_readiness.get('reason', '')}"
|
||||
)
|
||||
for attempt in signal_attempts:
|
||||
counts = attempt.get("counts") or {}
|
||||
layers = attempt.get("layers") or {}
|
||||
lines.append(
|
||||
" signal attempt "
|
||||
f"{attempt.get('attempt', '?')}: passed={attempt.get('passed', False)} "
|
||||
f"server_events={counts.get('server_events', 0)} "
|
||||
f"video={counts.get('video_events', 0)} audio={counts.get('audio_events', 0)} "
|
||||
f"paired={counts.get('paired_events', counts.get('coded_pairs', 0))} "
|
||||
f"audio_ready={layers.get('tethys_audio_ready', False)} "
|
||||
f"video_ready={layers.get('tethys_video_ready', False)} "
|
||||
f"reason={attempt.get('reason', '')}"
|
||||
)
|
||||
if "seed_video_delay_us" in result or "seed_audio_delay_us" in result:
|
||||
lines.append(
|
||||
" tuning: "
|
||||
@ -1549,7 +1802,7 @@ echo " ↪ capture_stack=${REMOTE_CAPTURE_STACK} audio_source=${REMOTE_AUDIO_S
|
||||
echo " ↪ tune_delays=${LESAVKA_SERVER_RC_TUNE_DELAYS} confirm=${LESAVKA_SERVER_RC_TUNE_CONFIRM} min_pairs=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS} max_abs_skew_ms=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS} max_step_us=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US} min_change_us=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}"
|
||||
echo " ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS} min_pairs=${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}"
|
||||
echo " ↪ coded_pairs_min=${LESAVKA_SERVER_RC_MIN_CODED_PAIRS} require_all_coded=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS} smoothness_gate=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS}"
|
||||
echo " ↪ signal_ready=${LESAVKA_SERVER_RC_SIGNAL_READY} min_pairs=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} duration=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}s warmup=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}s"
|
||||
echo " ↪ signal_ready=${LESAVKA_SERVER_RC_SIGNAL_READY} attempts=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS} min_pairs=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} duration=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}s warmup=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}s retry_delay=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s"
|
||||
echo " ↪ reconfigure=${LESAVKA_SERVER_RC_RECONFIGURE} strategy=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY} allow_gadget_reset=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET}"
|
||||
echo " ↪ tethys_ready=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY} settle=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}s timeout=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}s preroll_discard=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}s"
|
||||
echo " ↪ start_delay=${LESAVKA_SERVER_RC_START_DELAY_SECONDS}s"
|
||||
@ -1573,6 +1826,7 @@ for mode in "${modes[@]}"; do
|
||||
seed_result="${mode_dir}/mode-result-seed.json"
|
||||
readiness_dir="${mode_dir}/signal-readiness"
|
||||
readiness_log="${readiness_dir}/signal-readiness-run.log"
|
||||
readiness_attempts_json="${readiness_dir}/signal-readiness-attempts.json"
|
||||
tuned_log="${mode_dir}/mode-tuned-run.log"
|
||||
tuned_result="${mode_dir}/mode-result-tuned.json"
|
||||
tune_env="${mode_dir}/mode-tune-candidate.env"
|
||||
@ -1585,28 +1839,55 @@ for mode in "${modes[@]}"; do
|
||||
if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" ]]; then
|
||||
mkdir -p "${readiness_dir}"
|
||||
echo "==> mode ${mode}: proving Tethys signal readiness before measured probe"
|
||||
run_mode_probe \
|
||||
"${width}" \
|
||||
"${height}" \
|
||||
"${fps}" \
|
||||
"${audio_delay_us}" \
|
||||
"${video_delay_us}" \
|
||||
"${readiness_dir}" \
|
||||
"${readiness_log}" \
|
||||
"${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}" \
|
||||
"${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}" \
|
||||
"${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}"
|
||||
readiness_status=${RUN_MODE_PROBE_STATUS}
|
||||
readiness_artifact_dir="$(artifact_dir_from_log "${readiness_log}" "${readiness_dir}")"
|
||||
readiness_reason="probe command exited ${readiness_status}"
|
||||
if [[ "${readiness_status}" -eq 0 ]]; then
|
||||
readiness_pass=1
|
||||
readiness_status=0
|
||||
readiness_artifact_dir=""
|
||||
readiness_reason="no readiness attempts ran"
|
||||
readiness_attempt_jsons=()
|
||||
for readiness_attempt in $(seq 1 "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}"); do
|
||||
attempt_dir="${readiness_dir}/attempt-${readiness_attempt}"
|
||||
readiness_log="${attempt_dir}/signal-readiness-run.log"
|
||||
readiness_attempt_json="${attempt_dir}/signal-readiness-attempt.json"
|
||||
mkdir -p "${attempt_dir}"
|
||||
echo " ↪ readiness attempt ${readiness_attempt}/${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}: requiring ${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} paired coded events"
|
||||
run_mode_probe \
|
||||
"${width}" \
|
||||
"${height}" \
|
||||
"${fps}" \
|
||||
"${audio_delay_us}" \
|
||||
"${video_delay_us}" \
|
||||
"${attempt_dir}" \
|
||||
"${readiness_log}" \
|
||||
"${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}" \
|
||||
"${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}" \
|
||||
"${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}"
|
||||
readiness_status=${RUN_MODE_PROBE_STATUS}
|
||||
readiness_artifact_dir="$(artifact_dir_from_log "${readiness_log}" "${attempt_dir}")"
|
||||
set +e
|
||||
readiness_reason="$(signal_readiness_passed "${readiness_artifact_dir}" "${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" 2>&1)"
|
||||
readiness_reason="$(
|
||||
write_signal_readiness_attempt_result \
|
||||
"${readiness_attempt}" \
|
||||
"${readiness_artifact_dir}" \
|
||||
"${readiness_status}" \
|
||||
"${readiness_log}" \
|
||||
"${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \
|
||||
"${readiness_attempt_json}" 2>&1
|
||||
)"
|
||||
readiness_pass=$?
|
||||
set -e
|
||||
else
|
||||
readiness_pass=1
|
||||
fi
|
||||
readiness_attempt_jsons+=("${readiness_attempt_json}")
|
||||
if [[ "${readiness_pass}" -eq 0 ]]; then
|
||||
echo " ↪ readiness attempt ${readiness_attempt} passed: ${readiness_reason}"
|
||||
break
|
||||
fi
|
||||
[[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed"
|
||||
echo " ↪ readiness attempt ${readiness_attempt} failed: ${readiness_reason}"
|
||||
if [[ "${readiness_attempt}" -lt "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}" ]]; then
|
||||
echo " ↪ waiting ${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s before retrying signal readiness"
|
||||
sleep "${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}"
|
||||
fi
|
||||
done
|
||||
write_signal_readiness_attempts_summary "${readiness_attempts_json}" "${readiness_attempt_jsons[@]}"
|
||||
if [[ "${readiness_pass}" -ne 0 ]]; then
|
||||
[[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed"
|
||||
echo " ↪ signal readiness failed: ${readiness_reason}"
|
||||
@ -1620,6 +1901,7 @@ for mode in "${modes[@]}"; do
|
||||
"${readiness_status}" \
|
||||
"${readiness_log}" \
|
||||
"${readiness_artifact_dir}" \
|
||||
"${readiness_attempts_json}" \
|
||||
"${readiness_reason}" \
|
||||
"${mode_result}"
|
||||
cp "${mode_result}" "${seed_result}"
|
||||
|
||||
@ -90,6 +90,7 @@ REMOTE_CAPTURE=${REMOTE_CAPTURE:-"/tmp/lesavka-output-delay-probe-${STAMP}.mkv"}
|
||||
LOCAL_REPORT_DIR="${LOCAL_OUTPUT_DIR%/}/lesavka-output-delay-probe-${STAMP}"
|
||||
LOCAL_CAPTURE="${LOCAL_REPORT_DIR}/capture.mkv"
|
||||
LOCAL_ANALYSIS_JSON="${LOCAL_REPORT_DIR}/report.json"
|
||||
LOCAL_ANALYSIS_ERROR_LOG="${LOCAL_REPORT_DIR}/analysis-error.log"
|
||||
LOCAL_REPORT_TXT="${LOCAL_REPORT_DIR}/report.txt"
|
||||
LOCAL_EVENTS_CSV="${LOCAL_REPORT_DIR}/events.csv"
|
||||
LOCAL_SERVER_PROBE_REPLY="${LOCAL_REPORT_DIR}/server-output-probe-reply.txt"
|
||||
@ -532,9 +533,10 @@ retry_remote_artifact_command() {
|
||||
|
||||
run_remote_sync_analysis_once() {
|
||||
local output_path=$1
|
||||
local error_path=${2:-/dev/null}
|
||||
ssh ${SSH_OPTS} "${TETHYS_HOST}" \
|
||||
"chmod +x '${REMOTE_ANALYZE_BIN}' && '${REMOTE_ANALYZE_BIN}' '${remote_fetch_capture}' --json --event-width-codes '${PROBE_EVENT_WIDTH_CODES}' ${analysis_window_arg}" \
|
||||
> "${output_path}"
|
||||
> "${output_path}" 2> "${error_path}"
|
||||
}
|
||||
|
||||
resolve_server_addr() {
|
||||
@ -3350,16 +3352,28 @@ REMOTE_NORMALIZE_SCRIPT
|
||||
fi
|
||||
echo "==> analyzing capture on ${TETHYS_HOST}"
|
||||
analysis_tmp="${LOCAL_ANALYSIS_JSON}.tmp"
|
||||
analysis_error_tmp="${LOCAL_ANALYSIS_ERROR_LOG}.tmp"
|
||||
rm -f "${analysis_tmp}" "${analysis_error_tmp}" "${LOCAL_ANALYSIS_ERROR_LOG}"
|
||||
set +e
|
||||
retry_remote_artifact_command \
|
||||
"running remote sync analysis on ${TETHYS_HOST}" \
|
||||
run_remote_sync_analysis_once "${analysis_tmp}"
|
||||
run_remote_sync_analysis_once "${analysis_tmp}" "${analysis_error_tmp}"
|
||||
analysis_status=$?
|
||||
set -e
|
||||
if [[ "${analysis_status}" -eq 0 ]]; then
|
||||
mv "${analysis_tmp}" "${LOCAL_ANALYSIS_JSON}"
|
||||
if [[ -s "${analysis_error_tmp}" ]]; then
|
||||
mv "${analysis_error_tmp}" "${LOCAL_ANALYSIS_ERROR_LOG}"
|
||||
else
|
||||
rm -f "${analysis_error_tmp}"
|
||||
fi
|
||||
else
|
||||
rm -f "${analysis_tmp}"
|
||||
if [[ -s "${analysis_error_tmp}" ]]; then
|
||||
mv "${analysis_error_tmp}" "${LOCAL_ANALYSIS_ERROR_LOG}"
|
||||
else
|
||||
rm -f "${analysis_error_tmp}"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -3505,6 +3519,9 @@ fi
|
||||
if [[ -f "${LOCAL_ANALYSIS_JSON}" ]]; then
|
||||
echo "report_json: ${LOCAL_ANALYSIS_JSON}"
|
||||
fi
|
||||
if [[ -f "${LOCAL_ANALYSIS_ERROR_LOG}" ]]; then
|
||||
echo "analysis_error_log: ${LOCAL_ANALYSIS_ERROR_LOG}"
|
||||
fi
|
||||
if [[ -f "${LOCAL_REPORT_TXT}" ]]; then
|
||||
echo "report_txt: ${LOCAL_REPORT_TXT}"
|
||||
fi
|
||||
|
||||
@ -10,7 +10,7 @@ bench = false
|
||||
|
||||
[package]
|
||||
name = "lesavka_server"
|
||||
version = "0.19.25"
|
||||
version = "0.19.26"
|
||||
edition = "2024"
|
||||
autobins = false
|
||||
|
||||
|
||||
@ -33,6 +33,7 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
|
||||
"LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}",
|
||||
"LOCAL_REPORT_DIR=\"${LOCAL_OUTPUT_DIR%/}/lesavka-output-delay-probe-${STAMP}\"",
|
||||
"LOCAL_ANALYSIS_JSON=\"${LOCAL_REPORT_DIR}/report.json\"",
|
||||
"LOCAL_ANALYSIS_ERROR_LOG=\"${LOCAL_REPORT_DIR}/analysis-error.log\"",
|
||||
"LOCAL_EVENTS_CSV=\"${LOCAL_REPORT_DIR}/events.csv\"",
|
||||
"LOCAL_SERVER_PROBE_REPLY=\"${LOCAL_REPORT_DIR}/server-output-probe-reply.txt\"",
|
||||
"LOCAL_SERVER_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/server-output-timeline.json\"",
|
||||
@ -194,6 +195,8 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
|
||||
"fetching capture from ${TETHYS_HOST}",
|
||||
"warning: failed to fetch capture artifact; continuing with remote analysis JSON when available",
|
||||
"analysis_tmp=\"${LOCAL_ANALYSIS_JSON}.tmp\"",
|
||||
"analysis_error_tmp=\"${LOCAL_ANALYSIS_ERROR_LOG}.tmp\"",
|
||||
"analysis_error_log: ${LOCAL_ANALYSIS_ERROR_LOG}",
|
||||
"event_visibility",
|
||||
"Coded event visibility",
|
||||
"video_only",
|
||||
@ -420,9 +423,19 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() {
|
||||
"LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS:-0}",
|
||||
"LESAVKA_SERVER_RC_SIGNAL_READY=${LESAVKA_SERVER_RC_SIGNAL_READY:-1}",
|
||||
"LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS:-3}",
|
||||
"LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS:-12}",
|
||||
"LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS:-4}",
|
||||
"LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS:-5}",
|
||||
"signal_readiness_passed",
|
||||
"write_signal_readiness_attempt_result",
|
||||
"write_signal_readiness_attempts_summary",
|
||||
"schema\": \"lesavka.server-rc-signal-readiness-attempt.v1\"",
|
||||
"schema\": \"lesavka.server-rc-signal-readiness-summary.v1\"",
|
||||
"proving Tethys signal readiness before measured probe",
|
||||
"readiness attempt ${readiness_attempt}/${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}",
|
||||
"waiting ${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s before retrying signal readiness",
|
||||
"signal readiness did not pass",
|
||||
"signal attempt ",
|
||||
"smoothness_required",
|
||||
"smoothness_warnings",
|
||||
"smoothness warning:",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user