lesavka/scripts/manual/run_hardware_media_smoke.sh

#!/usr/bin/env bash
# scripts/manual/run_hardware_media_smoke.sh
# Manual: local/remote hardware media smoke evidence; not part of CI.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
STAMP="$(date +%Y%m%d-%H%M%S)"
ARTIFACT_DIR="${LESAVKA_HARDWARE_SMOKE_DIR:-/tmp/lesavka-hardware-media-smoke-${STAMP}}"
RESULTS_TSV="${ARTIFACT_DIR}/results.tsv"
SUMMARY_JSON="${ARTIFACT_DIR}/summary.json"
SUMMARY_TXT="${ARTIFACT_DIR}/summary.txt"

UPSTREAM_HEVC_FILE="${ARTIFACT_DIR}/upstream-hevc-nvenc.hevc"
UPSTREAM_HEVC_FRAME="${ARTIFACT_DIR}/upstream-hevc-cuda-frame.png"
DOWNSTREAM_H264_FILE="${ARTIFACT_DIR}/downstream-h264-nvenc.h264"
DOWNSTREAM_H264_FRAME="${ARTIFACT_DIR}/downstream-h264-cuda-frame.png"
AUDIO_WAV="${ARTIFACT_DIR}/audio-aac-roundtrip.wav"
AUDIO_RMS_JSON="${ARTIFACT_DIR}/audio-aac-roundtrip-rms.json"
SMOKE_WIDTH="${LESAVKA_HARDWARE_SMOKE_WIDTH:-1280}"
SMOKE_HEIGHT="${LESAVKA_HARDWARE_SMOKE_HEIGHT:-720}"
SMOKE_FPS="${LESAVKA_HARDWARE_SMOKE_FPS:-30}"
SMOKE_FRAMES="${LESAVKA_HARDWARE_SMOKE_FRAMES:-90}"
SMOKE_BITRATE_KBPS="${LESAVKA_HARDWARE_SMOKE_BITRATE_KBPS:-5000}"

mkdir -p "${ARTIFACT_DIR}"
: >"${RESULTS_TSV}"

OVERALL=0

sanitize() {
  printf '%s' "$*" | tr '\t\r\n' '   '
}

record_result() {
  local name="$1"
  local status="$2"
  local detail="$3"
  local artifact="${4:-}"
  printf '%s\t%s\t%s\t%s\n' \
    "${name}" \
    "${status}" \
    "$(sanitize "${detail}")" \
    "${artifact}" >>"${RESULTS_TSV}"
}

have_command() {
  command -v "$1" >/dev/null 2>&1
}

gst_has() {
  gst-inspect-1.0 "$1" >/dev/null 2>&1
}

ffmpeg_has_encoder() {
  local codec="$1"
  ffmpeg -hide_banner -encoders 2>/dev/null \
    | awk -v codec="${codec}" '$2 == codec { found = 1 } END { exit found ? 0 : 1 }'
}

ffmpeg_has_decoder() {
  local codec="$1"
  ffmpeg -hide_banner -decoders 2>/dev/null \
    | awk -v codec="${codec}" '$2 == codec { found = 1 } END { exit found ? 0 : 1 }'
}

run_logged() {
  local name="$1"
  local detail="$2"
  local rc=0
  shift 2
  local log="${ARTIFACT_DIR}/${name}.log"

  echo "==> ${name}"
  if "$@" >"${log}" 2>&1; then
    record_result "${name}" "pass" "${detail}" "${log}"
    echo "   pass"
    return 0
  else
    rc=$?
  fi

  record_result "${name}" "fail" "${detail}; exit=${rc}" "${log}"
  echo "   fail (exit ${rc})"
  sed -n '1,120p' "${log}" | sed 's/^/   | /'
  OVERALL=1
  return 0
}

run_shell() {
  local name="$1"
  local detail="$2"
  local command_text="$3"
  run_logged "${name}" "${detail}" bash -o pipefail -c "${command_text}"
}

finish_summary() {
  python3 - "${RESULTS_TSV}" "${SUMMARY_JSON}" "${SUMMARY_TXT}" "${ARTIFACT_DIR}" \
    "${UPSTREAM_HEVC_FILE}" "${UPSTREAM_HEVC_FRAME}" \
    "${DOWNSTREAM_H264_FILE}" "${DOWNSTREAM_H264_FRAME}" \
    "${AUDIO_WAV}" "${AUDIO_RMS_JSON}" <<'PY'
import json
import pathlib
import sys

results_tsv = pathlib.Path(sys.argv[1])
summary_json = pathlib.Path(sys.argv[2])
summary_txt = pathlib.Path(sys.argv[3])
artifact_dir = pathlib.Path(sys.argv[4])
artifact_paths = {
    "upstream_hevc_stream": sys.argv[5],
    "upstream_hevc_cuda_frame": sys.argv[6],
    "downstream_h264_stream": sys.argv[7],
    "downstream_h264_cuda_frame": sys.argv[8],
    "audio_aac_roundtrip_wav": sys.argv[9],
    "audio_aac_roundtrip_rms": sys.argv[10],
}

results = []
for line in results_tsv.read_text(encoding="utf-8").splitlines():
    name, status, detail, artifact = (line.split("\t") + ["", "", "", ""])[:4]
    results.append(
        {
            "name": name,
            "status": status,
            "detail": detail,
            "artifact": artifact,
        }
    )

failed = [row for row in results if row["status"] == "fail"]
summary = {
    "status": "fail" if failed else "pass",
    "artifact_dir": str(artifact_dir),
    "results": results,
    "artifacts": artifact_paths,
}
summary_json.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")

lines = [
    "Lesavka hardware media smoke summary",
    f"status: {summary['status']}",
    f"artifact_dir: {artifact_dir}",
    "",
]
for row in results:
    artifact = f" ({row['artifact']})" if row["artifact"] else ""
    lines.append(f"- {row['status']}: {row['name']} - {row['detail']}{artifact}")
lines.extend(["", "Inspectable artifacts:"])
for name, path in artifact_paths.items():
    lines.append(f"- {name}: {path}")
summary_txt.write_text("\n".join(lines) + "\n", encoding="utf-8")
PY

  echo "==> hardware media smoke summary"
  sed -n '1,160p' "${SUMMARY_TXT}"
  echo "summary_json: ${SUMMARY_JSON}"
  echo "summary_txt: ${SUMMARY_TXT}"
}

require_prereqs() {
  local missing=()
  for cmd in ffmpeg gst-inspect-1.0 gst-launch-1.0 python3 awk grep; do
    if ! have_command "${cmd}"; then
      missing+=("${cmd}")
    fi
  done

  if ((${#missing[@]})); then
    record_result "prerequisites" "fail" "missing commands: ${missing[*]}" ""
    OVERALL=1
    return 1
  fi

  record_result "prerequisites" "pass" "ffmpeg, GStreamer, and Python are available" ""
  return 0
}

select_gst_h264_decoder() {
  local decoder
  for decoder in nvh264dec nvh264sldec vah264dec vaapih264dec vaapi264dec v4l2h264dec v4l2slh264dec; do
    if gst_has "${decoder}"; then
      printf '%s\n' "${decoder}"
      return 0
    fi
  done
  if [[ ${LESAVKA_ALLOW_VULKAN_H264_DECODER:-0} == 1 ]] && gst_has vulkanh264dec; then
    printf '%s\n' vulkanh264dec
    return 0
  fi
  return 1
}

gst_h264_decode_chain() {
  case "$1" in
    vulkanh264dec)
      printf '%s\n' 'vulkanh264dec discard-corrupted-frames=true automatic-request-sync-points=true ! vulkandownload'
      ;;
    nvh264sldec|nvh264dec|vah264dec|vaapih264dec|vaapi264dec|v4l2h264dec|v4l2slh264dec)
      printf '%s\n' "$1"
      ;;
    *)
      return 1
      ;;
  esac
}

select_gst_aac_encoder() {
  local encoder
  for encoder in fdkaacenc voaacenc avenc_aac; do
    if gst_has "${encoder}"; then
      printf '%s\n' "${encoder}"
      return 0
    fi
  done
  return 1
}

main() {
  echo "==> Lesavka hardware media smoke"
  echo "   artifact_dir=${ARTIFACT_DIR}"
  echo "   video=${SMOKE_WIDTH}x${SMOKE_HEIGHT}@${SMOKE_FPS} frames=${SMOKE_FRAMES}"
  echo "   no sudo, no systemctl, no UVC gadget reset, and no display-manager reset are used"

  if ! require_prereqs; then
    finish_summary
    exit 1
  fi

  if ! ffmpeg_has_encoder hevc_nvenc; then
    record_result "client_upstream_hevc_nvenc_available" "fail" "ffmpeg hevc_nvenc encoder is missing" ""
    OVERALL=1
  else
    record_result "client_upstream_hevc_nvenc_available" "pass" "ffmpeg hevc_nvenc encoder is available" ""
  fi

  if ! ffmpeg_has_encoder h264_nvenc; then
    record_result "client_downstream_h264_nvenc_source_available" "fail" "ffmpeg h264_nvenc encoder is missing for downstream test source" ""
    OVERALL=1
  else
    record_result "client_downstream_h264_nvenc_source_available" "pass" "ffmpeg h264_nvenc encoder is available for downstream test source" ""
  fi

  if ! ffmpeg_has_decoder hevc_cuvid; then
    record_result "client_hevc_cuvid_visual_decode_available" "fail" "ffmpeg hevc_cuvid decoder is missing for visual evidence frame" ""
    OVERALL=1
  else
    record_result "client_hevc_cuvid_visual_decode_available" "pass" "ffmpeg hevc_cuvid decoder is available for visual evidence frame" ""
  fi

  if ! ffmpeg_has_decoder h264_cuvid; then
    record_result "client_h264_cuvid_visual_decode_available" "fail" "ffmpeg h264_cuvid decoder is missing for visual evidence frame" ""
    OVERALL=1
  else
    record_result "client_h264_cuvid_visual_decode_available" "pass" "ffmpeg h264_cuvid decoder is available for visual evidence frame" ""
  fi

  local gst_h264_decoder=""
  local gst_h264_chain=""
  if gst_h264_decoder="$(select_gst_h264_decoder)"; then
    gst_h264_chain="$(gst_h264_decode_chain "${gst_h264_decoder}")"
    record_result "client_downstream_gstreamer_h264_hw_decoder_available" "pass" "selected ${gst_h264_decoder}" ""
  else
    record_result "client_downstream_gstreamer_h264_hw_decoder_available" "fail" "no GStreamer hardware H.264 decoder found" ""
    OVERALL=1
  fi

  local aac_encoder=""
  if aac_encoder="$(select_gst_aac_encoder)" && gst_has avdec_aac && gst_has wavenc; then
    record_result "audio_roundtrip_elements_available" "pass" "selected ${aac_encoder} -> avdec_aac -> wavenc" ""
  else
    record_result "audio_roundtrip_elements_available" "fail" "AAC encoder/decoder/wavenc elements are missing" ""
    OVERALL=1
  fi

  if [[ ${OVERALL} -eq 0 ]]; then
    run_shell "client_upstream_hevc_nvenc_file" \
      "GPU encodes a synthetic upstream HEVC stream with hevc_nvenc" \
      "ffmpeg -hide_banner -loglevel warning -y -nostdin -f lavfi -i testsrc2=size=${SMOKE_WIDTH}x${SMOKE_HEIGHT}:rate=${SMOKE_FPS} -frames:v ${SMOKE_FRAMES} -an -sn -dn -vf format=nv12 -c:v hevc_nvenc -preset p1 -tune ll -rc cbr -b:v ${SMOKE_BITRATE_KBPS}k -maxrate ${SMOKE_BITRATE_KBPS}k -bufsize ${SMOKE_BITRATE_KBPS}k -g ${SMOKE_FPS} -bf 0 -forced-idr 1 -f hevc '${UPSTREAM_HEVC_FILE}'"

    run_shell "client_upstream_hevc_gstreamer_parse" \
      "GStreamer accepts the HEVC elementary stream shape used by the upstream bundle path" \
      "gst-launch-1.0 -q filesrc location='${UPSTREAM_HEVC_FILE}' ! h265parse config-interval=-1 ! video/x-h265,stream-format=byte-stream,alignment=au ! fakesink sync=false"

    run_shell "client_upstream_hevc_cuvid_frame" \
      "CUDA decodes one visual proof frame from the upstream HEVC stream" \
      "ffmpeg -hide_banner -loglevel warning -y -nostdin -c:v hevc_cuvid -i '${UPSTREAM_HEVC_FILE}' -frames:v 1 '${UPSTREAM_HEVC_FRAME}'"

    run_shell "client_downstream_h264_nvenc_file" \
      "GPU creates a downstream-like H.264 elementary stream for decoder verification" \
      "ffmpeg -hide_banner -loglevel warning -y -nostdin -f lavfi -i testsrc2=size=${SMOKE_WIDTH}x${SMOKE_HEIGHT}:rate=${SMOKE_FPS} -frames:v ${SMOKE_FRAMES} -an -sn -dn -vf format=nv12 -c:v h264_nvenc -preset p1 -tune ll -rc cbr -b:v ${SMOKE_BITRATE_KBPS}k -maxrate ${SMOKE_BITRATE_KBPS}k -bufsize ${SMOKE_BITRATE_KBPS}k -g ${SMOKE_FPS} -bf 0 -forced-idr 1 -f h264 '${DOWNSTREAM_H264_FILE}'"

    run_shell "client_downstream_h264_gstreamer_hw_decode" \
      "GStreamer decodes H.264 with hardware decoder ${gst_h264_decoder}" \
      "gst-launch-1.0 -q filesrc location='${DOWNSTREAM_H264_FILE}' ! h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au ! ${gst_h264_chain} ! videoconvert ! fakesink sync=false"

    run_shell "client_downstream_h264_cuvid_frame" \
      "CUDA decodes one visual proof frame from the downstream H.264 stream" \
      "ffmpeg -hide_banner -loglevel warning -y -nostdin -c:v h264_cuvid -i '${DOWNSTREAM_H264_FILE}' -frames:v 1 '${DOWNSTREAM_H264_FRAME}'"

    run_shell "audio_aac_roundtrip_wav" \
      "GStreamer encodes and decodes a 1 kHz tone to a WAV artifact for audio-path sanity" \
      "gst-launch-1.0 -q audiotestsrc wave=sine freq=1000 num-buffers=120 ! audio/x-raw,format=S16LE,channels=2,rate=48000 ! audioconvert ! audioresample ! ${aac_encoder} ! aacparse ! avdec_aac ! audioconvert ! audioresample ! audio/x-raw,format=S16LE,channels=2,rate=48000 ! wavenc ! filesink location='${AUDIO_WAV}'"

    run_shell "audio_aac_roundtrip_rms" \
      "Decoded WAV contains non-silent audio energy" \
      "python3 - '${AUDIO_WAV}' '${AUDIO_RMS_JSON}' <<'PY'
import json
import math
import pathlib
import struct
import sys
import wave

wav_path = pathlib.Path(sys.argv[1])
out_path = pathlib.Path(sys.argv[2])
with wave.open(str(wav_path), 'rb') as wav:
    frames = wav.readframes(wav.getnframes())
    sample_count = len(frames) // 2
    samples = struct.unpack('<' + 'h' * sample_count, frames)
    rms = math.sqrt(sum(sample * sample for sample in samples) / max(sample_count, 1))
summary = {'rms': rms, 'sample_count': sample_count, 'wav': str(wav_path)}
out_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + '\\n', encoding='utf-8')
if rms < 500:
    raise SystemExit(f'audio RMS too low: {rms:.2f}')
print(json.dumps(summary, sort_keys=True))
PY"
  fi

  if [[ ${LESAVKA_RUN_REMOTE_MEDIA_SMOKE:-0} == 1 ]]; then
    local remote_host="${LESAVKA_SERVER_HOST:-titan-jh}"
    local ssh_opts="${SSH_OPTS:--o BatchMode=yes -o ConnectTimeout=5}"
    if ssh ${ssh_opts} "${remote_host}" "gst-inspect-1.0 v4l2slh265dec >/dev/null"; then
      run_shell "server_hevc_hardware_decode" \
        "Remote server decodes the same upstream HEVC stream with v4l2slh265dec; no service/gadget mutation" \
        "cat '${UPSTREAM_HEVC_FILE}' | ssh ${ssh_opts} '${remote_host}' \"gst-launch-1.0 -q fdsrc blocksize=1048576 ! h265parse disable-passthrough=true config-interval=-1 ! video/x-h265,stream-format=byte-stream,alignment=au ! v4l2slh265dec discard-corrupted-frames=true automatic-request-sync-points=true ! videoconvert ! fakesink sync=false\""
    else
      record_result "server_hevc_hardware_decode" "fail" "remote ${remote_host} did not expose v4l2slh265dec over ssh" ""
      OVERALL=1
    fi
  else
    record_result "server_hevc_hardware_decode" "skipped" "set LESAVKA_RUN_REMOTE_MEDIA_SMOKE=1 to verify Theia's hardware HEVC decoder without touching services" ""
  fi

  finish_summary
  exit "${OVERALL}"
}

main "$@"