"""Failure-stage, subprocess cleanup, and metadata privacy regression tests.""" import json from pathlib import Path import signal import sys import threading import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "services/hermes/scripts")) import suite_backends import suite_multipass from suite_cli_diagnostics import snapshot from suite_contract import CLAUDE_MAX_TURNS, MODELS, Problem, preflight, validate_request from suite_jobs import Jobs MODEL = MODELS["claude"]["model"] CANARY = "PRIVATE_SOURCE_OR_PROVIDER_ERROR_MUST_NOT_ESCAPE" def envelope(**changes): """Construct synthetic native CLI events, with content canaries.""" init = {"type": "system", "subtype": "init", "model": MODEL, "tools": ["StructuredOutput"], "mcp_servers": [], "plugins": []} final = {"type": "result", "subtype": "success", "is_error": False, "num_turns": 2, "stop_reason": "tool_use", "usage": {"input_tokens": 120, "output_tokens": 50, CANARY: CANARY}, "modelUsage": {MODEL: {"contextWindow": 1000000, "maxOutputTokens": 64000, "canonicalModel": MODEL, "provider": "firstParty", CANARY: CANARY}}, "structured_output": {"groups": [{"name": "Synthetic", "description": CANARY, "members": ["CASE-1"]}]}, "errors": [CANARY], "result": CANARY, **changes} return "\n".join(json.dumps(event) for event in (init, final)) def request(): """Force the approved Claude route with synthetic input too large for local.""" return validate_request({"campaign": "SYNTHETIC", "suite": "DIAGNOSTICS", "cases": [{"alias": "CASE-1", "description": "Synthetic example. " * 500}], "routing": {"allow_external": True, "allowed_external_providers": ["claude"]}}, ["claude"]) @pytest.mark.parametrize("subtype,code", [ ("error_max_turns", "incomplete_generation"), ("error_max_structured_output_retries", "incomplete_generation"), ("error_max_budget_usd", "job_cost_budget_exhausted"), ("error_during_execution", "incomplete_generation"), (CANARY, "incomplete_generation"), ]) def test_unsuccessful_final_is_specific_and_content_free(subtype, code): raw = envelope(subtype=subtype, is_error=True, num_turns=4, structured_output=None) with pytest.raises(Problem, match=code) as raised: suite_backends.parse_claude(raw, MODEL, exit_code=1, subprocess_timeout_seconds=899.5) details = raised.value.details assert details["failure_stage"] == "cli_final_result" assert details["exit_code"] == 1 and details["turns"] == 4 assert details["max_turns"] == CLAUDE_MAX_TURNS == 6 assert details["turn_limit_reached"] == (None if subtype == CANARY else subtype == "error_max_turns") assert details["structured_retry_limit_reached"] == (None if subtype == CANARY else subtype == "error_max_structured_output_retries") assert details["usage"] == {"input_tokens": 120, "output_tokens": 50} assert details["structured_output_present"] is False assert CANARY not in json.dumps(raised.value.document()) @pytest.mark.parametrize("status,code", [(429, "rate_limit"), (401, "provider_authentication"), (403, "provider_authentication")]) def test_provider_error_status_is_preserved(status, code): with pytest.raises(Problem, match=code) as raised: suite_backends.parse_claude(envelope(subtype="error_during_execution", is_error=True, api_error_status=status), MODEL, exit_code=1) assert raised.value.details["api_error_status"] == status @pytest.mark.parametrize("category,code", [ ("authentication_failed", "provider_authentication"), ("oauth_org_not_allowed", "provider_authentication"), ("rate_limit", "rate_limit"), ("server_error", "backend_unavailable"), ("overloaded", "backend_unavailable"), ("unknown", "incomplete_generation"), (CANARY, "incomplete_generation"), ]) def test_error_flag_on_success_subtype_uses_safe_assistant_category(category, code): """A result subtype alone does not mean the CLI completed successfully.""" assistant = {"type": "assistant", "error": category, "is_api_error_message": True, "message": {"content": [{"type": "text", "text": CANARY}]}} raw = json.dumps(assistant) + "\n" + envelope(is_error=True, structured_output=None, num_turns=1, usage={"output_tokens": 0}) with pytest.raises(Problem, match=code) as raised: suite_backends.parse_claude(raw, MODEL, exit_code=1, reasoning_effort="high") details = raised.value.details assert details["final_event_subtype"] == "success" and details["final_is_error"] is True assert details["assistant_error_code"] == ("other" if category == CANARY else category) assert details["assistant_api_error_seen"] is True assert details["failure_stage"] == "cli_final_result" and details["turns"] == 1 assert details["turn_limit_reached"] is False and details["reasoning_effort"] == "high" assert CANARY not in json.dumps(raised.value.document()) @pytest.mark.parametrize("text,marked,transport,code", [ ("API Error: Connection error.", True, "connection_error", "backend_unavailable"), ("API Error: Connection refused \u2014 a firewall or proxy may be blocking it (ECONNREFUSED)", True, "connection_refused", "backend_unavailable"), ("API Error: Request timed out.", True, "request_timeout", "backend_timeout_or_unavailable"), ("API Error: Connection error.", False, None, "incomplete_generation"), ("API Error: Connection error. " + CANARY, True, None, "incomplete_generation"), ]) def test_transport_signatures_require_exact_cli_error_marker(text, marked, transport, code): """Provider text never becomes a diagnostic string or a substring classifier.""" assistant = {"type": "assistant", "error": "unknown", "is_api_error_message": marked, "message": {"content": [{"type": "text", "text": text}]}} raw = json.dumps(assistant) + "\n" + envelope(is_error=True, structured_output=None) with pytest.raises(Problem, match=code) as raised: suite_backends.parse_claude(raw, MODEL, exit_code=1) assert raised.value.details["cli_transport_error"] == transport assert CANARY not in json.dumps(raised.value.document()) @pytest.mark.parametrize("effort", ["high", "xhigh", None]) def test_reasoning_effort_reports_actual_configuration_or_unknown(effort): assert snapshot(envelope(), reasoning_effort=effort)["reasoning_effort"] == effort @pytest.mark.parametrize("reason", ["max_tokens", "model_context_window_exceeded"]) def test_explicit_provider_stop_distinct_from_turn_limit(reason): with pytest.raises(Problem, match="incomplete_generation") as raised: suite_backends.parse_claude(envelope(stop_reason=reason), MODEL, exit_code=0) assert raised.value.details["failure_stage"] == "provider_stop" assert raised.value.details["provider_stop_reason"] == reason assert raised.value.details["turn_limit_reached"] is False @pytest.mark.parametrize("raw,stage,code", [ ("", "missing_init_or_final_event", "incomplete_generation"), ('{"type":"system","subtype":"init","model":"claude-opus-4-8"}', "missing_init_or_final_event", "incomplete_generation"), ("not json " + CANARY, "cli_event_json", "invalid_json_result"), ("[]", "cli_event_json", "invalid_json_result"), ]) def test_missing_final_and_malformed_stream_are_distinct(raw, stage, code): with pytest.raises(Problem, match=code) as raised: suite_backends.parse_claude(raw, MODEL, exit_code=0) assert raised.value.details["failure_stage"] == stage assert CANARY not in json.dumps(raised.value.document()) def test_text_or_tool_candidate_is_detected_but_not_silently_accepted(): value = json.loads(envelope().splitlines()[-1])["structured_output"] assistant = {"type": "assistant", "message": {"stop_reason": "end_turn", "content": [ {"type": "tool_use", "name": "StructuredOutput", "input": value}, {"type": "text", "text": json.dumps(value)}]}} raw = json.dumps(assistant) + "\n" + envelope(structured_output=None, result=json.dumps(value)) with pytest.raises(Problem, match="invalid_json_result") as raised: suite_backends.parse_claude(raw, MODEL, exit_code=0) details = raised.value.details assert details["failure_stage"] == "structured_output_extraction" assert details["assistant_structured_tool_input_present"] is True assert details["assistant_json_text_present"] is True assert details["final_json_text_present"] is True assert CANARY not in json.dumps(details) def test_valid_structured_result_and_exit_status_both_checked(): raw = envelope() result, metadata = suite_backends.parse_claude(raw, MODEL, exit_code=0) assert result["groups"][0]["description"] == CANARY assert CANARY not in json.dumps(metadata) for code in (1, -signal.SIGKILL): with pytest.raises(Problem, match="incomplete_generation") as raised: suite_backends.parse_claude(raw, MODEL, exit_code=code) assert raised.value.details["failure_stage"] == "process_exit" assert raised.value.details["structured_output_present"] is True assert raised.value.details["termination_signal"] == (signal.SIGKILL if code < 0 else None) def test_unknown_measurements_never_become_content_or_fabricated_zeroes(): details = snapshot(envelope(num_turns=CANARY, usage={"input_tokens": True, "output_tokens": float("nan")}, duration_api_ms=CANARY, total_cost_usd=float("inf"), stop_reason=CANARY)) assert details["turns"] is None and details["duration_api_ms"] is None assert details["usage"] == {"input_tokens": None, "output_tokens": None} assert details["provider_stop_reason"] == "other" assert details["provider_timeout_seconds"] is None and details["reasoning_token_limit"] is None assert CANARY not in json.dumps(details, allow_nan=False) @pytest.mark.parametrize("mutation,code", [("schema", "invalid_json_result"), ("coverage", "invalid_case_assignments")]) def test_validation_failure_retains_usage_without_accepting_content(tmp_path, monkeypatch, capsys, mutation, code): value = request() selected = preflight(value) result, metadata = suite_backends.parse_claude(envelope(), MODEL, exit_code=0) if mutation == "schema": result["groups"][0]["description"] = "x" * 241 else: result["groups"][0]["members"].append("CASE-1") monkeypatch.setattr(suite_backends, "switchyard_decision", lambda *_: None) metadata["review_summary"] = {} monkeypatch.setattr(suite_multipass, "generate", lambda *_: (result, metadata)) jobs = Jobs(tmp_path / "jobs.sqlite") job, _ = jobs.submit("owner", "synthetic-validation", value, selected, "192.168.22.8", launch=False) jobs.run(job["job_id"], "owner", value, selected, "192.168.22.8") failed = jobs.get(job["job_id"], "owner") assert failed["error"]["code"] == code assert failed["error"]["details"]["failure_stage"] == "result_validation" assert failed["usage"]["output_tokens"] == 50 and failed["turns"] == 2 assert not jobs.results assert CANARY not in json.dumps(failed) + capsys.readouterr().out @pytest.mark.parametrize("mode,code,stage", [ ("success", None, None), ("nonzero", "incomplete_generation", "process_exit"), ("signal", "incomplete_generation", "process_exit"), ("empty", "backend_unavailable", "process_exit"), ("timeout", "timeout", "subprocess"), ("cancel", "cancelled", "subprocess"), ("oversize", "response_too_large", "subprocess"), ("start", "backend_unavailable", "process_start"), ]) def test_actual_subprocess_paths_cleanup_and_safe_metadata(tmp_path, monkeypatch, mode, code, stage): real_tempdir = suite_backends.tempfile.TemporaryDirectory real_read = Path.read_text monkeypatch.setattr(suite_backends.tempfile, "TemporaryDirectory", lambda **_: real_tempdir(prefix="suite-", dir=tmp_path)) monkeypatch.setattr(Path, "read_text", lambda p, *a, **k: "fake-token" if str(p) == "/vault/secrets/claude-token" else real_read(p, *a, **k)) script = "import os,signal,time; print(" + repr(envelope()) + ",flush=True); " if mode in {"timeout", "cancel"}: script += "time.sleep(30)" elif mode == "signal": script += "os.kill(os.getpid(),signal.SIGKILL)" elif mode == "nonzero": script += "raise SystemExit(1)" elif mode == "empty": script = "raise SystemExit(1)" elif mode == "oversize": monkeypatch.setattr(suite_backends, "OUTPUT_BYTES", 64) elif mode == "success": script += "time.sleep(0.25)" command = ["/not-installed-synthetic-cli"] if mode == "start" else [sys.executable, "-c", script] monkeypatch.setattr(suite_backends, "claude_command", lambda *_, **__: command) value, cancel = request(), threading.Event() if mode == "timeout": value["execution"]["max_seconds"] = 0.02 if mode == "cancel": cancel.set() if code: with pytest.raises(Problem, match=code) as raised: suite_backends.claude_generate(value, cancel) details = raised.value.details assert details["failure_stage"] == stage assert details["reasoning_effort"] == MODELS["claude"]["reasoning"] if mode in {"timeout", "cancel"}: assert details["termination_signal"] == signal.SIGTERM assert details["termination_reason"] == mode.replace("cancel", "cancelled") assert CANARY not in json.dumps(details) else: updates = [] _, metadata = suite_backends.claude_generate(value, cancel, progress=updates.append) assert metadata["cli_diagnostics"]["exit_code"] == 0 assert metadata["cli_diagnostics"]["reasoning_effort"] == MODELS["claude"]["reasoning"] assert metadata["temporary_files_deleted"] is True assert updates and updates[-1]["cli_output_bytes"] > 0 assert updates[-1]["cli_running"] is True assert CANARY not in json.dumps(updates) assert list(tmp_path.iterdir()) == [] def test_failed_cli_usage_survives_job_cleanup_without_content(tmp_path, monkeypatch, capsys): value, jobs = request(), Jobs(tmp_path / "jobs.sqlite") selected = preflight(value) monkeypatch.setattr(suite_backends, "switchyard_decision", lambda *_: None) def fail(*_, **kwargs): suite_backends.parse_claude(envelope(subtype="error_max_turns", is_error=True, structured_output=None), MODEL, exit_code=1) monkeypatch.setattr(suite_backends, "claude_generate", fail) document, _ = jobs.submit("owner", "synthetic-failure", value, selected, "192.168.22.8", launch=False) jobs.run(document["job_id"], "owner", value, selected, "192.168.22.8") failed = jobs.get(document["job_id"], "owner") assert failed["error"]["details"]["turn_limit_reached"] is True assert failed["usage"]["output_tokens"] == 50 assert not jobs.results and not jobs.active replay, created = jobs.submit("owner", "synthetic-failure", value, selected, "192.168.22.8", launch=False) assert not created and replay["job_id"] == document["job_id"] assert CANARY not in json.dumps(failed) + capsys.readouterr().out def test_process_exit_race_still_reaps(monkeypatch): from types import SimpleNamespace waited = [] def exited(*_): raise ProcessLookupError() monkeypatch.setattr(suite_backends.os, "killpg", exited) process = SimpleNamespace(pid=1, poll=lambda: None, wait=lambda **_: waited.append(True)) suite_backends.stop(process) assert waited == [True]