229 lines
7.2 KiB
Bash
Executable File
229 lines
7.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Run opt-in hardware/lab gates that are unsafe for shared CI desktops.
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
|
REPORT_DIR="${ROOT_DIR}/target/baremetal-lab-gate"
|
|
SUMMARY_JSON="${REPORT_DIR}/summary.json"
|
|
SUMMARY_TXT="${REPORT_DIR}/summary.txt"
|
|
METRICS_FILE="${REPORT_DIR}/metrics.prom"
|
|
RUN_LOG="${REPORT_DIR}/baremetal-lab-gate.log"
|
|
PUSHGATEWAY_URL=${QUALITY_GATE_PUSHGATEWAY_URL:-}
|
|
PUSHGATEWAY_JOB=${LESAVKA_LAB_GATE_PUSHGATEWAY_JOB:-lesavka-baremetal-lab-gate}
|
|
|
|
mkdir -p "${REPORT_DIR}"
|
|
cd "${ROOT_DIR}"
|
|
|
|
branch=${BRANCH_NAME:-${GIT_BRANCH:-}}
|
|
if [[ -z "${branch}" ]]; then
|
|
branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)
|
|
fi
|
|
commit=${GIT_COMMIT:-}
|
|
if [[ -z "${commit}" ]]; then
|
|
commit=$(git rev-parse --short HEAD 2>/dev/null || echo unknown)
|
|
fi
|
|
build_url=${BUILD_URL:-}
|
|
|
|
log() {
|
|
printf '%s\n' "$*" | tee -a "${RUN_LOG}"
|
|
}
|
|
|
|
: >"${RUN_LOG}"
|
|
|
|
status=0
|
|
outcome=ok
|
|
detail="ran configured bare-metal lab gates"
|
|
steps_jsonl="${REPORT_DIR}/steps.jsonl"
|
|
: >"${steps_jsonl}"
|
|
start_seconds=$(date +%s)
|
|
|
|
record_step() {
|
|
local name="$1"
|
|
local result="$2"
|
|
local note="$3"
|
|
python3 - "${steps_jsonl}" "${name}" "${result}" "${note}" <<'PY'
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
path = pathlib.Path(sys.argv[1])
|
|
entry = {'name': sys.argv[2], 'result': sys.argv[3], 'note': sys.argv[4]}
|
|
with path.open('a', encoding='utf-8') as fh:
|
|
fh.write(json.dumps(entry, sort_keys=True) + '\n')
|
|
PY
|
|
}
|
|
|
|
run_shell_step() {
|
|
local name="$1"
|
|
local command="$2"
|
|
log "==> ${name}"
|
|
set +e
|
|
bash -lc "${command}" 2>&1 | tee -a "${RUN_LOG}"
|
|
local step_status=${PIPESTATUS[0]}
|
|
set -e
|
|
if [[ "${step_status}" -eq 0 ]]; then
|
|
record_step "${name}" ok "completed"
|
|
else
|
|
record_step "${name}" failed "exit ${step_status}"
|
|
status="${step_status}"
|
|
fi
|
|
}
|
|
|
|
run_step() {
|
|
local name="$1"
|
|
shift
|
|
log "==> ${name}"
|
|
set +e
|
|
"$@" 2>&1 | tee -a "${RUN_LOG}"
|
|
local step_status=${PIPESTATUS[0]}
|
|
set -e
|
|
if [[ "${step_status}" -eq 0 ]]; then
|
|
record_step "${name}" ok "completed"
|
|
else
|
|
record_step "${name}" failed "exit ${step_status}"
|
|
status="${step_status}"
|
|
fi
|
|
}
|
|
|
|
if [[ "${LESAVKA_ALLOW_LAB_HARDWARE_TESTS:-0}" != "1" ]]; then
|
|
outcome=skipped
|
|
detail="bare-metal lab gates require LESAVKA_ALLOW_LAB_HARDWARE_TESTS=1"
|
|
log "Skipping bare-metal lab gates."
|
|
log "These gates may use Theia/Tethys, UVC/UAC devices, or virtual HID input."
|
|
log "Run only on an isolated worker/session with LESAVKA_ALLOW_LAB_HARDWARE_TESTS=1."
|
|
else
|
|
if [[ "${LESAVKA_RUN_VIDEO_DOWNSTREAM_GATE:-1}" == "1" ]]; then
|
|
run_step "video_downstream_gate" scripts/ci/video_downstream_gate.sh
|
|
else
|
|
record_step "video_downstream_gate" skipped "LESAVKA_RUN_VIDEO_DOWNSTREAM_GATE!=1"
|
|
fi
|
|
|
|
if [[ "${LESAVKA_ALLOW_DISRUPTIVE_INPUT_TESTS:-0}" == "1" ]]; then
|
|
run_step "input_transport_gate" scripts/ci/input_transport_gate.sh
|
|
else
|
|
record_step "input_transport_gate" skipped "LESAVKA_ALLOW_DISRUPTIVE_INPUT_TESTS!=1"
|
|
log "Skipping disruptive input transport gate; set LESAVKA_ALLOW_DISRUPTIVE_INPUT_TESTS=1 on an isolated worker."
|
|
fi
|
|
|
|
if [[ "${LESAVKA_RUN_SERVER_RCT_MATRIX:-0}" == "1" ]]; then
|
|
server_rct_matrix_cmd=${LESAVKA_SERVER_RCT_MATRIX_CMD:-"${ROOT_DIR}/scripts/manual/run_server_to_rc_mode_matrix.sh"}
|
|
run_shell_step "server_to_rct_matrix" "${server_rct_matrix_cmd}"
|
|
else
|
|
record_step "server_to_rct_matrix" skipped "LESAVKA_RUN_SERVER_RCT_MATRIX!=1"
|
|
fi
|
|
|
|
if [[ "${LESAVKA_RUN_CLIENT_RCT_PROBE:-0}" == "1" ]]; then
|
|
client_rct_probe_cmd=${LESAVKA_CLIENT_RCT_PROBE_CMD:-"${ROOT_DIR}/scripts/manual/run_client_to_rct_transport_probe.sh"}
|
|
run_shell_step "client_to_rct_transport_probe" "${client_rct_probe_cmd}"
|
|
else
|
|
record_step "client_to_rct_transport_probe" skipped "LESAVKA_RUN_CLIENT_RCT_PROBE!=1"
|
|
fi
|
|
|
|
if [[ "${status}" -ne 0 ]]; then
|
|
outcome=failed
|
|
detail="one or more bare-metal lab gates failed"
|
|
fi
|
|
fi
|
|
|
|
duration_seconds=$(($(date +%s) - start_seconds))
|
|
|
|
python3 - \
|
|
"${SUMMARY_JSON}" \
|
|
"${SUMMARY_TXT}" \
|
|
"${METRICS_FILE}" \
|
|
"${steps_jsonl}" \
|
|
"${branch}" \
|
|
"${commit}" \
|
|
"${build_url}" \
|
|
"${outcome}" \
|
|
"${status}" \
|
|
"${duration_seconds}" \
|
|
"${detail}" <<'PY'
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
summary_path = pathlib.Path(sys.argv[1])
|
|
text_path = pathlib.Path(sys.argv[2])
|
|
metrics_path = pathlib.Path(sys.argv[3])
|
|
steps_path = pathlib.Path(sys.argv[4])
|
|
branch = sys.argv[5]
|
|
commit = sys.argv[6]
|
|
build_url = sys.argv[7]
|
|
outcome = sys.argv[8]
|
|
status = int(sys.argv[9])
|
|
duration_seconds = int(sys.argv[10])
|
|
detail = sys.argv[11]
|
|
|
|
def esc(value: str) -> str:
|
|
return value.replace('\\', r'\\').replace('\n', r'\n').replace('"', r'\"')
|
|
|
|
steps = []
|
|
if steps_path.exists():
|
|
for line in steps_path.read_text(encoding='utf-8').splitlines():
|
|
if line.strip():
|
|
steps.append(json.loads(line))
|
|
|
|
summary = {
|
|
'suite': 'lesavka',
|
|
'profile': 'lab',
|
|
'branch': branch,
|
|
'commit': commit,
|
|
'build_url': build_url,
|
|
'outcome': outcome,
|
|
'exit_code': status,
|
|
'duration_seconds': duration_seconds,
|
|
'detail': detail,
|
|
'steps': steps,
|
|
'generated_at': datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + '\n', encoding='utf-8')
|
|
text_path.write_text(
|
|
'\n'.join([
|
|
f'lesavka bare-metal lab gate: {outcome}',
|
|
f'branch: {branch}',
|
|
f'commit: {commit}',
|
|
f'duration: {duration_seconds}s',
|
|
f'detail: {detail}',
|
|
'steps:',
|
|
*[f"- {step['name']}: {step['result']} ({step['note']})" for step in steps],
|
|
]) + '\n',
|
|
encoding='utf-8',
|
|
)
|
|
|
|
labels = f'suite="lesavka",profile="lab",branch="{esc(branch)}",commit="{esc(commit)}"'
|
|
ok = 1 if outcome == 'ok' else 0
|
|
failed = 1 if outcome == 'failed' else 0
|
|
skipped = 1 if outcome == 'skipped' else 0
|
|
lines = [
|
|
'# HELP lesavka_ci_profile_last_run_success Whether the latest Lesavka CI profile run succeeded.',
|
|
'# TYPE lesavka_ci_profile_last_run_success gauge',
|
|
f'lesavka_ci_profile_last_run_success{{{labels}}} {ok}',
|
|
'# HELP lesavka_ci_profile_duration_seconds Duration of the latest Lesavka CI profile run.',
|
|
'# TYPE lesavka_ci_profile_duration_seconds gauge',
|
|
f'lesavka_ci_profile_duration_seconds{{{labels}}} {duration_seconds}',
|
|
'# HELP lesavka_ci_profile_runs Current profile run outcome.',
|
|
'# TYPE lesavka_ci_profile_runs gauge',
|
|
f'lesavka_ci_profile_runs{{{labels},status="ok"}} {ok}',
|
|
f'lesavka_ci_profile_runs{{{labels},status="failed"}} {failed}',
|
|
f'lesavka_ci_profile_runs{{{labels},status="skipped"}} {skipped}',
|
|
'# HELP lesavka_lab_gate_step_result Bare-metal lab gate step result.',
|
|
'# TYPE lesavka_lab_gate_step_result gauge',
|
|
]
|
|
for step in steps:
|
|
name = esc(step['name'])
|
|
for result in ['ok', 'failed', 'skipped']:
|
|
value = 1 if step['result'] == result else 0
|
|
lines.append(f'lesavka_lab_gate_step_result{{{labels},step="{name}",result="{result}"}} {value}')
|
|
metrics_path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
|
PY
|
|
|
|
if [[ -n "${PUSHGATEWAY_URL}" ]]; then
|
|
curl --fail --silent --show-error \
|
|
--data-binary @"${METRICS_FILE}" \
|
|
"${PUSHGATEWAY_URL%/}/metrics/job/${PUSHGATEWAY_JOB}/suite/lesavka" || status=$?
|
|
fi
|
|
|
|
exit "${status}"
|