From dc4b5f6e8b18387c003a17c4d5f4396e9e672a23 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Mon, 4 May 2026 21:38:30 -0300 Subject: [PATCH] test(server-rc): retry remote artifact transfer --- Cargo.lock | 6 +- client/Cargo.toml | 2 +- common/Cargo.toml | 2 +- scripts/manual/run_upstream_av_sync.sh | 64 +++++++++++++++++-- server/Cargo.toml | 2 +- .../client_manual_sync_script_contract.rs | 10 +++ 6 files changed, 74 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d02cfd5..2b52589 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.19.23" +version = "0.19.24" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.19.23" +version = "0.19.24" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.19.23" +version = "0.19.24" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 7f03450..ae98612 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.19.23" +version = "0.19.24" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 9124e7f..1bd808e 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.19.23" +version = "0.19.24" edition = "2024" build = "build.rs" diff --git a/scripts/manual/run_upstream_av_sync.sh b/scripts/manual/run_upstream_av_sync.sh index 472f49c..45fd7c3 100755 --- a/scripts/manual/run_upstream_av_sync.sh +++ b/scripts/manual/run_upstream_av_sync.sh @@ -58,6 +58,8 @@ REMOTE_ANALYZE=${REMOTE_ANALYZE:-1} REMOTE_ANALYZE_BIN=${REMOTE_ANALYZE_BIN:-/tmp/lesavka-sync-analyze} REMOTE_ANALYZE_COPY=${REMOTE_ANALYZE_COPY:-1} FETCH_CAPTURE=${FETCH_CAPTURE:-1} +REMOTE_ARTIFACT_RETRIES=${REMOTE_ARTIFACT_RETRIES:-3} +REMOTE_ARTIFACT_RETRY_DELAY_SECONDS=${REMOTE_ARTIFACT_RETRY_DELAY_SECONDS:-5} REMOTE_SERVER_PREFLIGHT=${REMOTE_SERVER_PREFLIGHT:-1} REMOTE_EXPECT_CAM_OUTPUT=${REMOTE_EXPECT_CAM_OUTPUT:-uvc} REMOTE_EXPECT_UVC_CODEC=${REMOTE_EXPECT_UVC_CODEC:-mjpeg} @@ -505,6 +507,36 @@ start_server_tunnel() { wait_for_server_tunnel "${local_port}" } +retry_remote_artifact_command() { + local description=$1 + shift + local attempt status + for attempt in $(seq 1 "${REMOTE_ARTIFACT_RETRIES}"); do + set +e + "$@" + status=$? + set -e + if [[ "${status}" -eq 0 ]]; then + return 0 + fi + if [[ "${attempt}" -lt "${REMOTE_ARTIFACT_RETRIES}" ]]; then + printf '%s failed with status %s; retrying %s/%s in %ss\n' \ + "${description}" "${status}" "$((attempt + 1))" "${REMOTE_ARTIFACT_RETRIES}" \ + "${REMOTE_ARTIFACT_RETRY_DELAY_SECONDS}" >&2 + sleep "${REMOTE_ARTIFACT_RETRY_DELAY_SECONDS}" + fi + done + printf '%s failed after %s attempt(s)\n' "${description}" "${REMOTE_ARTIFACT_RETRIES}" >&2 + return "${status}" +} + +run_remote_sync_analysis_once() { + local output_path=$1 + ssh ${SSH_OPTS} "${TETHYS_HOST}" \ + "chmod +x '${REMOTE_ANALYZE_BIN}' && '${REMOTE_ANALYZE_BIN}' '${remote_fetch_capture}' --json --event-width-codes '${PROBE_EVENT_WIDTH_CODES}' ${analysis_window_arg}" \ + > "${output_path}" +} + resolve_server_addr() { if [[ "${LESAVKA_SERVER_ADDR}" != "auto" ]]; then RESOLVED_LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_ADDR}" @@ -3140,7 +3172,9 @@ if [[ -f "${LOCAL_CAPTURE_LOG}" ]] \ capture_streamon_timeout=1 fi -if ssh ${SSH_OPTS} "${TETHYS_HOST}" "test -f '${REMOTE_CAPTURE}'"; then +if retry_remote_artifact_command \ + "checking remote capture on ${TETHYS_HOST}" \ + ssh ${SSH_OPTS} "${TETHYS_HOST}" "test -f '${REMOTE_CAPTURE}'"; then remote_fetch_capture="${REMOTE_CAPTURE}" analysis_status=0 if [[ "${ANALYSIS_NORMALIZE}" != "0" ]]; then @@ -3173,24 +3207,42 @@ REMOTE_NORMALIZE_SCRIPT if [[ "${REMOTE_ANALYZE}" != "0" ]]; then if [[ "${REMOTE_ANALYZE_COPY}" != "0" ]]; then echo "==> copying sync analyzer to ${TETHYS_HOST}:${REMOTE_ANALYZE_BIN}" - scp ${SSH_OPTS} "${ANALYZE_BIN}" "${TETHYS_HOST}:${REMOTE_ANALYZE_BIN}" + retry_remote_artifact_command \ + "copying sync analyzer to ${TETHYS_HOST}" \ + scp ${SSH_OPTS} "${ANALYZE_BIN}" "${TETHYS_HOST}:${REMOTE_ANALYZE_BIN}" fi analysis_window_arg="$(compute_analysis_window_arg)" if [[ -n "${analysis_window_arg}" ]]; then echo " ↪ analyzer timeline window: ${analysis_window_arg#--analysis-window-s }" fi echo "==> analyzing capture on ${TETHYS_HOST}" + analysis_tmp="${LOCAL_ANALYSIS_JSON}.tmp" set +e - ssh ${SSH_OPTS} "${TETHYS_HOST}" \ - "chmod +x '${REMOTE_ANALYZE_BIN}' && '${REMOTE_ANALYZE_BIN}' '${remote_fetch_capture}' --json --event-width-codes '${PROBE_EVENT_WIDTH_CODES}' ${analysis_window_arg}" \ - > "${LOCAL_ANALYSIS_JSON}" + retry_remote_artifact_command \ + "running remote sync analysis on ${TETHYS_HOST}" \ + run_remote_sync_analysis_once "${analysis_tmp}" analysis_status=$? set -e + if [[ "${analysis_status}" -eq 0 ]]; then + mv "${analysis_tmp}" "${LOCAL_ANALYSIS_JSON}" + else + rm -f "${analysis_tmp}" + fi fi if [[ "${FETCH_CAPTURE}" != "0" ]]; then echo "==> fetching capture back to ${LOCAL_CAPTURE}" - scp ${SSH_OPTS} "${TETHYS_HOST}:${remote_fetch_capture}" "${LOCAL_CAPTURE}" + fetch_capture_status=0 + retry_remote_artifact_command \ + "fetching capture from ${TETHYS_HOST}" \ + scp ${SSH_OPTS} "${TETHYS_HOST}:${remote_fetch_capture}" "${LOCAL_CAPTURE}" \ + || fetch_capture_status=$? + if [[ "${fetch_capture_status}" -ne 0 ]]; then + echo "warning: failed to fetch capture artifact; continuing with remote analysis JSON when available" >&2 + if [[ "${REMOTE_ANALYZE}" == "0" ]]; then + exit "${fetch_capture_status}" + fi + fi fi if [[ "${analysis_status}" -ne 0 ]]; then echo "remote analysis failed with status ${analysis_status}; capture preserved at ${LOCAL_CAPTURE}" >&2 diff --git a/server/Cargo.toml b/server/Cargo.toml index 8340826..15ef26c 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.19.23" +version = "0.19.24" edition = "2024" autobins = false diff --git a/testing/tests/client_manual_sync_script_contract.rs b/testing/tests/client_manual_sync_script_contract.rs index cdc376a..ebaf834 100644 --- a/testing/tests/client_manual_sync_script_contract.rs +++ b/testing/tests/client_manual_sync_script_contract.rs @@ -184,6 +184,16 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() { "\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"", "REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}", "REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}", + "REMOTE_ARTIFACT_RETRIES=${REMOTE_ARTIFACT_RETRIES:-3}", + "REMOTE_ARTIFACT_RETRY_DELAY_SECONDS=${REMOTE_ARTIFACT_RETRY_DELAY_SECONDS:-5}", + "retry_remote_artifact_command", + "run_remote_sync_analysis_once", + "checking remote capture on ${TETHYS_HOST}", + "copying sync analyzer to ${TETHYS_HOST}", + "running remote sync analysis on ${TETHYS_HOST}", + "fetching capture from ${TETHYS_HOST}", + "warning: failed to fetch capture artifact; continuing with remote analysis JSON when available", + "analysis_tmp=\"${LOCAL_ANALYSIS_JSON}.tmp\"", "REMOTE_PULSE_AUDIO_ANCHOR_SILENCE=${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE:-1}", "anchoring Pulse capture audio timeline with generated silence", "audiotestsrc wave=silence is-live=true do-timestamp=true",