fix(sync): tunnel manual probe through ssh

scripts/manual/run_upstream_av_sync.sh

@@ -50,6 +50,67 @@ LOCAL_CAPTURE="${LOCAL_OUTPUT_DIR}/lesavka-upstream-av-sync-${STAMP}.mkv"
 LOCAL_ANALYSIS_JSON="${LOCAL_CAPTURE%.mkv}.json"
 LOCAL_CAPTURE_LOG="${LOCAL_CAPTURE%.mkv}.capture.log"
 RESOLVED_LESAVKA_SERVER_ADDR=""
+SERVER_TUNNEL_PID=""
+SERVER_TUNNEL_REMOTE_PORT=""
+SERVER_TUNNEL_LOCAL_PORT=""
+
+cleanup_server_tunnel() {
+  if [[ -z "${SERVER_TUNNEL_PID}" ]]; then
+    return 0
+  fi
+  if kill -0 "${SERVER_TUNNEL_PID}" >/dev/null 2>&1; then
+    kill "${SERVER_TUNNEL_PID}" >/dev/null 2>&1 || true
+    wait "${SERVER_TUNNEL_PID}" >/dev/null 2>&1 || true
+  fi
+}
+
+trap cleanup_server_tunnel EXIT
+
+pick_local_server_tunnel_port() {
+  python3 - <<'PY'
+import socket
+
+with socket.socket() as sock:
+    sock.bind(("127.0.0.1", 0))
+    print(sock.getsockname()[1])
+PY
+}
+
+wait_for_server_tunnel() {
+  local local_port=$1
+  local tries=50
+  local i=0
+  while (( i < tries )); do
+    if nc -z 127.0.0.1 "${local_port}" >/dev/null 2>&1; then
+      return 0
+    fi
+    if ! kill -0 "${SERVER_TUNNEL_PID}" >/dev/null 2>&1; then
+      wait "${SERVER_TUNNEL_PID}" >/dev/null 2>&1 || true
+      echo "SSH tunnel to ${LESAVKA_SERVER_HOST} exited before becoming ready" >&2
+      exit 88
+    fi
+    sleep 0.1
+    ((i += 1))
+  done
+
+  echo "SSH tunnel to ${LESAVKA_SERVER_HOST} did not become ready on localhost:${local_port}" >&2
+  exit 89
+}
+
+start_server_tunnel() {
+  local remote_port=$1
+  local local_port
+  local_port="$(pick_local_server_tunnel_port)"
+  echo "==> opening SSH tunnel to ${LESAVKA_SERVER_HOST}:127.0.0.1:${remote_port} on localhost:${local_port}"
+  ssh ${SSH_OPTS} -o ExitOnForwardFailure=yes \
+    -N \
+    -L "127.0.0.1:${local_port}:127.0.0.1:${remote_port}" \
+    "${LESAVKA_SERVER_HOST}" &
+  SERVER_TUNNEL_PID=$!
+  SERVER_TUNNEL_REMOTE_PORT="${remote_port}"
+  SERVER_TUNNEL_LOCAL_PORT="${local_port}"
+  wait_for_server_tunnel "${local_port}"
+}
 
 resolve_server_addr() {
   if [[ "${LESAVKA_SERVER_ADDR}" != "auto" ]]; then
@@ -65,11 +126,13 @@ resolve_server_addr() {
   )"
   port="${bind_addr##*:}"
   if [[ "${port}" =~ ^[0-9]+$ ]]; then
-    RESOLVED_LESAVKA_SERVER_ADDR="http://${LESAVKA_SERVER_CONNECT_HOST}:${port}"
+    start_server_tunnel "${port}"
+    RESOLVED_LESAVKA_SERVER_ADDR="http://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}"
     return 0
   fi
 
-  RESOLVED_LESAVKA_SERVER_ADDR="http://${LESAVKA_SERVER_CONNECT_HOST}:50051"
+  start_server_tunnel "50051"
+  RESOLVED_LESAVKA_SERVER_ADDR="http://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}"
 }
 
 preflight_server_path() {
@@ -140,6 +203,9 @@ fi
 
 resolve_server_addr
 echo "==> resolved Lesavka server addr: ${RESOLVED_LESAVKA_SERVER_ADDR}"
+if [[ -n "${SERVER_TUNNEL_PID}" ]]; then
+  echo "   ↪ tunneled to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}"
+fi
 
 preflight_server_path
 
@@ -633,6 +699,14 @@ fi
 REMOTE_CAPTURE_SCRIPT
 capture_pid=$!
 
+sleep 1
+if ! kill -0 "${capture_pid}" >/dev/null 2>&1; then
+  capture_status=0
+  wait "${capture_pid}" || capture_status=$?
+  echo "Tethys capture failed before the sync probe could start; see ${LOCAL_CAPTURE_LOG} for details." >&2
+  exit "${capture_status}"
+fi
+
 sleep "${LEAD_IN_SECONDS}"
 
 echo "==> running local Lesavka sync probe against ${RESOLVED_LESAVKA_SERVER_ADDR}"

testing/tests/client_manual_sync_script_contract.rs

@@ -0,0 +1,33 @@
+//! Contract tests for the manual upstream A/V sync harness.
+//!
+//! Scope: statically guard the workstation-side tunnel/bootstrap behavior.
+//! Targets: `scripts/manual/run_upstream_av_sync.sh`.
+//! Why: the manual probe should reach Theia through SSH even when the gRPC
+//! port is not exposed on the public SSH endpoint.
+
+const SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_av_sync.sh");
+
+#[test]
+fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
+    for expected in [
+        "cleanup_server_tunnel",
+        "pick_local_server_tunnel_port",
+        "wait_for_server_tunnel",
+        "start_server_tunnel",
+        "ExitOnForwardFailure=yes",
+        "127.0.0.1:${local_port}:127.0.0.1:${remote_port}",
+        "RESOLVED_LESAVKA_SERVER_ADDR=\"http://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}\"",
+        "tunneled to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}",
+        "Tethys capture failed before the sync probe could start",
+        "kill -0 \"${capture_pid}\"",
+    ] {
+        assert!(
+            SYNC_SCRIPT.contains(expected),
+            "manual sync script should contain {expected}"
+        );
+    }
+    assert!(
+        !SYNC_SCRIPT.contains("RESOLVED_LESAVKA_SERVER_ADDR=\"http://${LESAVKA_SERVER_CONNECT_HOST}:${port}\""),
+        "auto server resolution should not guess a public gRPC host when SSH is already required"
+    );
+}