atlas-iac/testing/tests/test_suite_cli_diagnostics.py

"""Failure-stage, subprocess cleanup, and metadata privacy regression tests."""
import json
from pathlib import Path
import signal
import sys
import threading
import time

import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "services/hermes/scripts"))
import suite_backends
import suite_multipass
from suite_cli_diagnostics import snapshot
from suite_contract import CLAUDE_MAX_TURNS, CLAUDE_MODELS, MODELS, Problem, preflight, validate_request
from suite_jobs import Jobs

MODEL = MODELS["claude"]["model"]
CANARY = "PRIVATE_SOURCE_OR_PROVIDER_ERROR_MUST_NOT_ESCAPE"


def envelope(**changes):
    """Construct synthetic native CLI events, with content canaries."""
    init = {"type": "system", "subtype": "init", "model": MODEL,
            "tools": ["StructuredOutput"], "mcp_servers": [], "plugins": []}
    final = {"type": "result", "subtype": "success", "is_error": False,
             "num_turns": 2, "stop_reason": "tool_use",
             "usage": {"input_tokens": 120, "output_tokens": 50, CANARY: CANARY},
             "modelUsage": {MODEL: {"contextWindow": 1000000, "maxOutputTokens": CLAUDE_MODELS[MODEL],
                                     "canonicalModel": MODEL, "provider": "firstParty", CANARY: CANARY}},
             "structured_output": {"groups": [{"name": "Synthetic", "description": CANARY,
                                                 "members": ["CASE-1"]}]},
             "errors": [CANARY], "result": CANARY, **changes}
    return "\n".join(json.dumps(event) for event in (init, final))


def request():
    """Force the approved Claude route with synthetic input too large for local."""
    return validate_request({"campaign": "SYNTHETIC", "suite": "DIAGNOSTICS",
        "cases": [{"alias": "CASE-1", "description": "Synthetic example. " * 500}],
        "routing": {"allow_external": True, "allowed_external_providers": ["claude"]}}, ["claude"])


@pytest.mark.parametrize("subtype,code", [
    ("error_max_turns", "pass_turn_limit"),
    ("error_max_structured_output_retries", "pass_schema_repair"),
    ("error_max_budget_usd", "unexpected_cli_budget_limit"),
    ("error_during_execution", "incomplete_generation"),
    (CANARY, "incomplete_generation"),
])
def test_unsuccessful_final_is_specific_and_content_free(subtype, code):
    raw = envelope(subtype=subtype, is_error=True, num_turns=4, structured_output=None)
    with pytest.raises(Problem, match=code) as raised:
        suite_backends.parse_claude(raw, MODEL, exit_code=1, subprocess_timeout_seconds=899.5)
    details = raised.value.details
    assert details["failure_stage"] == "cli_final_result"
    assert details["exit_code"] == 1 and details["turns"] == 4
    assert details["max_turns"] == CLAUDE_MAX_TURNS == 6
    assert details["turn_limit_reached"] == (None if subtype == CANARY else subtype == "error_max_turns")
    assert details["structured_retry_limit_reached"] == (None if subtype == CANARY else subtype == "error_max_structured_output_retries")
    assert details["usage"] == {"input_tokens": 120, "output_tokens": 50}
    assert details["structured_output_present"] is False
    assert CANARY not in json.dumps(raised.value.document())


@pytest.mark.parametrize("status,code", [(429, "rate_limit"), (401, "provider_authentication"), (403, "provider_authentication")])
def test_provider_error_status_is_preserved(status, code):
    with pytest.raises(Problem, match=code) as raised:
        suite_backends.parse_claude(envelope(subtype="error_during_execution", is_error=True,
                                            api_error_status=status), MODEL, exit_code=1)
    assert raised.value.details["api_error_status"] == status


@pytest.mark.parametrize("category,code", [
    ("authentication_failed", "provider_authentication"),
    ("oauth_org_not_allowed", "provider_authentication"),
    ("rate_limit", "rate_limit"), ("server_error", "backend_unavailable"),
    ("overloaded", "backend_unavailable"), ("unknown", "pass_provider_transient_failure"),
    (CANARY, "pass_provider_transient_failure"),
])
def test_error_flag_on_success_subtype_uses_safe_assistant_category(category, code):
    """A result subtype alone does not mean the CLI completed successfully."""
    assistant = {"type": "assistant", "error": category, "is_api_error_message": True,
                 "message": {"content": [{"type": "text", "text": CANARY}]}}
    raw = json.dumps(assistant) + "\n" + envelope(is_error=True, structured_output=None,
                                                  num_turns=1, usage={"output_tokens": 0})
    with pytest.raises(Problem, match=code) as raised:
        suite_backends.parse_claude(raw, MODEL, exit_code=1, reasoning_effort="high")
    details = raised.value.details
    assert details["final_event_subtype"] == "success" and details["final_is_error"] is True
    assert details["assistant_error_code"] == ("other" if category == CANARY else category)
    assert details["assistant_api_error_seen"] is True
    assert details["failure_stage"] == "cli_final_result" and details["turns"] == 1
    assert details["turn_limit_reached"] is False and details["reasoning_effort"] == "high"
    assert CANARY not in json.dumps(raised.value.document())


@pytest.mark.parametrize("text,marked,transport,code", [
    ("API Error: Connection error.", True, "connection_error", "backend_unavailable"),
    ("API Error: Connection refused \u2014 a firewall or proxy may be blocking it (ECONNREFUSED)",
     True, "connection_refused", "backend_unavailable"),
    ("API Error: Request timed out.", True, "request_timeout", "backend_timeout_or_unavailable"),
    ("API Error: Connection error.", False, None, "incomplete_generation"),
    ("API Error: Connection error. " + CANARY, True, None, "pass_provider_transient_failure"),
])
def test_transport_signatures_require_exact_cli_error_marker(text, marked, transport, code):
    """Provider text never becomes a diagnostic string or a substring classifier."""
    assistant = {"type": "assistant", "error": "unknown", "is_api_error_message": marked,
                 "message": {"content": [{"type": "text", "text": text}]}}
    raw = json.dumps(assistant) + "\n" + envelope(is_error=True, structured_output=None)
    with pytest.raises(Problem, match=code) as raised:
        suite_backends.parse_claude(raw, MODEL, exit_code=1)
    assert raised.value.details["cli_transport_error"] == transport
    assert CANARY not in json.dumps(raised.value.document())


@pytest.mark.parametrize("effort", ["high", "xhigh", None])
def test_reasoning_effort_reports_actual_configuration_or_unknown(effort):
    assert snapshot(envelope(), reasoning_effort=effort)["reasoning_effort"] == effort


@pytest.mark.parametrize("reason", ["max_tokens", "model_context_window_exceeded"])
def test_explicit_provider_stop_distinct_from_turn_limit(reason):
    with pytest.raises(Problem, match="pass_output_limit|pass_context_limit") as raised:
        suite_backends.parse_claude(envelope(stop_reason=reason), MODEL, exit_code=0)
    assert raised.value.details["failure_stage"] == "provider_stop"
    assert raised.value.details["provider_stop_reason"] == reason
    assert raised.value.details["turn_limit_reached"] is False


@pytest.mark.parametrize("raw,stage,code", [
    ("", "missing_init_or_final_event", "pass_missing_terminal_event"),
    (json.dumps({"type": "system", "subtype": "init", "model": MODEL}), "missing_init_or_final_event", "pass_missing_terminal_event"),
    ("not json " + CANARY, "cli_event_json", "invalid_json_result"),
    ("[]", "cli_event_json", "invalid_json_result"),
])
def test_missing_final_and_malformed_stream_are_distinct(raw, stage, code):
    with pytest.raises(Problem, match=code) as raised:
        suite_backends.parse_claude(raw, MODEL, exit_code=0)
    assert raised.value.details["failure_stage"] == stage
    assert CANARY not in json.dumps(raised.value.document())


def test_text_or_tool_candidate_is_detected_but_not_silently_accepted():
    value = json.loads(envelope().splitlines()[-1])["structured_output"]
    assistant = {"type": "assistant", "message": {"stop_reason": "end_turn", "content": [
        {"type": "tool_use", "name": "StructuredOutput", "input": value},
        {"type": "text", "text": json.dumps(value)}]}}
    raw = json.dumps(assistant) + "\n" + envelope(structured_output=None, result=json.dumps(value))
    result, metadata = suite_backends.parse_claude(raw, MODEL, exit_code=0)
    assert result == value
    details = metadata['cli_diagnostics']
    assert details['structured_output_location'] == 'result.result_json'
    assert CANARY not in json.dumps(details)


def test_valid_structured_result_and_exit_status_both_checked():
    raw = envelope()
    result, metadata = suite_backends.parse_claude(raw, MODEL, exit_code=0)
    assert result["groups"][0]["description"] == CANARY
    assert CANARY not in json.dumps(metadata)
    for code in (1, -signal.SIGKILL):
        with pytest.raises(Problem, match="incomplete_generation") as raised:
            suite_backends.parse_claude(raw, MODEL, exit_code=code)
        assert raised.value.details["failure_stage"] == "process_exit"
        assert raised.value.details["structured_output_present"] is True
        assert raised.value.details["termination_signal"] == (signal.SIGKILL if code < 0 else None)


def test_unknown_measurements_never_become_content_or_fabricated_zeroes():
    details = snapshot(envelope(num_turns=CANARY, usage={"input_tokens": True, "output_tokens": float("nan")},
                                duration_api_ms=CANARY, total_cost_usd=float("inf"), stop_reason=CANARY))
    assert details["turns"] is None and details["duration_api_ms"] is None
    assert details["usage"] == {"input_tokens": None, "output_tokens": None}
    assert details["provider_stop_reason"] == "other"
    assert details["provider_timeout_seconds"] is None and details["reasoning_token_limit"] is None
    assert CANARY not in json.dumps(details, allow_nan=False)


@pytest.mark.parametrize("mutation,code", [("schema", "invalid_json_result"), ("coverage", "invalid_case_assignments")])
def test_validation_failure_retains_usage_without_accepting_content(tmp_path, monkeypatch, capsys, mutation, code):
    value = request()
    selected = preflight(value)
    result, metadata = suite_backends.parse_claude(envelope(), MODEL, exit_code=0)
    if mutation == "schema":
        result["groups"][0]["description"] = "x" * 241
    else:
        result["groups"][0]["members"].append("CASE-1")
    monkeypatch.setattr(suite_backends, "switchyard_decision", lambda *_: None)
    metadata["review_summary"] = {}
    monkeypatch.setattr(suite_multipass, "generate", lambda *_, **__: (result, metadata))
    jobs = Jobs(tmp_path / "jobs.sqlite")
    job, _ = jobs.submit("owner", "synthetic-validation", value, selected, "192.168.22.8", launch=False)
    jobs.run(job["job_id"], "owner", value, selected, "192.168.22.8")
    failed = jobs.get(job["job_id"], "owner")
    assert failed["error"]["code"] == code
    assert failed["error"]["details"]["failure_stage"] == "result_validation"
    assert failed["usage"]["output_tokens"] == 50 and failed["turns"] == 2
    assert not jobs.results
    assert CANARY not in json.dumps(failed) + capsys.readouterr().out


@pytest.mark.parametrize("mode,code,stage", [
    ("success", None, None), ("nonzero", "incomplete_generation", "process_exit"),
    ("signal", "incomplete_generation", "process_exit"),
    ("empty", "backend_unavailable", "process_exit"),
    ("timeout", "pass_timeout", "subprocess"), ("cancel", "cancelled", "subprocess"),
    ("job_timeout", "job_time_budget_exhausted", "subprocess"),
    ("oversize", "response_too_large", "subprocess"),
    ("start", "backend_unavailable", "process_start"),
])
def test_actual_subprocess_paths_cleanup_and_safe_metadata(tmp_path, monkeypatch, mode, code, stage):
    real_tempdir = suite_backends.tempfile.TemporaryDirectory
    real_read = Path.read_text
    monkeypatch.setattr(suite_backends.tempfile, "TemporaryDirectory",
                        lambda **_: real_tempdir(prefix="suite-", dir=tmp_path))
    monkeypatch.setattr(Path, "read_text", lambda p, *a, **k: "fake-token" if str(p) == "/vault/secrets/claude-token" else real_read(p, *a, **k))
    script = "import os,signal,time; print(" + repr(envelope()) + ",flush=True); "
    if mode in {"timeout", "job_timeout", "cancel"}:
        script += "time.sleep(30)"
    elif mode == "signal":
        script += "os.kill(os.getpid(),signal.SIGKILL)"
    elif mode == "nonzero":
        script += "raise SystemExit(1)"
    elif mode == "empty":
        script = "raise SystemExit(1)"
    elif mode == "oversize":
        monkeypatch.setattr(suite_backends, "OUTPUT_BYTES", 64)
    elif mode == "success":
        script += "time.sleep(0.25)"
    command = ["/not-installed-synthetic-cli"] if mode == "start" else [sys.executable, "-c", script]
    monkeypatch.setattr(suite_backends, "claude_command", lambda *_, **__: command)
    value, cancel = request(), threading.Event()
    if mode == "timeout":
        value["execution"]["max_seconds"] = 0.02
    if mode == "cancel":
        cancel.set()
    if code:
        with pytest.raises(Problem, match=code) as raised:
            suite_backends.claude_generate(value, cancel,
                job_deadline=time.monotonic() + 0.02 if mode == "job_timeout" else None)
        details = raised.value.details
        assert details["failure_stage"] == stage
        assert details["reasoning_effort"] == MODELS["claude"]["reasoning"]
        if mode in {"timeout", "job_timeout", "cancel"}:
            assert details["termination_signal"] == signal.SIGTERM
            assert details["termination_reason"] == code
        assert CANARY not in json.dumps(details)
    else:
        updates = []
        _, metadata = suite_backends.claude_generate(value, cancel, progress=updates.append)
        assert metadata["cli_diagnostics"]["exit_code"] == 0
        assert metadata["cli_diagnostics"]["reasoning_effort"] == MODELS["claude"]["reasoning"]
        assert metadata["temporary_files_deleted"] is True
        assert updates and updates[-1]["cli_output_bytes"] > 0
        assert updates[-1]["cli_running"] is True
        assert CANARY not in json.dumps(updates)
    assert list(tmp_path.iterdir()) == []


def test_failed_cli_usage_survives_job_cleanup_without_content(tmp_path, monkeypatch, capsys):
    value, jobs = request(), Jobs(tmp_path / "jobs.sqlite")
    selected = preflight(value)
    monkeypatch.setattr(suite_backends, "switchyard_decision", lambda *_: None)
    def fail(*_, **kwargs):
        suite_backends.parse_claude(envelope(subtype="error_max_turns", is_error=True,
                                             structured_output=None), MODEL, exit_code=1)
    monkeypatch.setattr(suite_backends, "claude_generate", fail)
    document, _ = jobs.submit("owner", "synthetic-failure", value, selected, "192.168.22.8", launch=False)
    jobs.run(document["job_id"], "owner", value, selected, "192.168.22.8")
    failed = jobs.get(document["job_id"], "owner")
    assert failed["error"]["details"]["turn_limit_reached"] is True
    assert failed["usage"]["output_tokens"] == 50 * len(failed["error"]["details"]["attempts_metadata"])
    assert not jobs.results and not jobs.active
    replay, created = jobs.submit("owner", "synthetic-failure", value, selected, "192.168.22.8", launch=False)
    assert not created and replay["job_id"] == document["job_id"]
    assert CANARY not in json.dumps(failed) + capsys.readouterr().out


def test_process_exit_race_still_reaps(monkeypatch):
    from types import SimpleNamespace
    waited = []
    def exited(*_):
        raise ProcessLookupError()
    monkeypatch.setattr(suite_backends.os, "killpg", exited)
    process = SimpleNamespace(pid=1, poll=lambda: None, wait=lambda **_: waited.append(True))
    suite_backends.stop(process)
    assert waited == [True]