239 lines
12 KiB
Python
239 lines
12 KiB
Python
"""Failure-stage, subprocess cleanup, and metadata privacy regression tests."""
|
|
import json
|
|
from pathlib import Path
|
|
import signal
|
|
import sys
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "services/hermes/scripts"))
|
|
import suite_backends
|
|
import suite_multipass
|
|
from suite_cli_diagnostics import snapshot
|
|
from suite_contract import CLAUDE_MAX_TURNS, MODELS, Problem, preflight, validate_request
|
|
from suite_jobs import Jobs
|
|
|
|
MODEL = MODELS["claude"]["model"]
|
|
CANARY = "PRIVATE_SOURCE_OR_PROVIDER_ERROR_MUST_NOT_ESCAPE"
|
|
|
|
|
|
def envelope(**changes):
|
|
"""Construct synthetic native CLI events, with content canaries."""
|
|
init = {"type": "system", "subtype": "init", "model": MODEL,
|
|
"tools": ["StructuredOutput"], "mcp_servers": [], "plugins": []}
|
|
final = {"type": "result", "subtype": "success", "is_error": False,
|
|
"num_turns": 2, "stop_reason": "tool_use",
|
|
"usage": {"input_tokens": 120, "output_tokens": 50, CANARY: CANARY},
|
|
"modelUsage": {MODEL: {"contextWindow": 1000000, "maxOutputTokens": 64000,
|
|
"canonicalModel": MODEL, "provider": "firstParty", CANARY: CANARY}},
|
|
"structured_output": {"groups": [{"name": "Synthetic", "description": CANARY,
|
|
"members": ["CASE-1"]}]},
|
|
"errors": [CANARY], "result": CANARY, **changes}
|
|
return "\n".join(json.dumps(event) for event in (init, final))
|
|
|
|
|
|
def request():
|
|
"""Force the approved Claude route with synthetic input too large for local."""
|
|
return validate_request({"campaign": "SYNTHETIC", "suite": "DIAGNOSTICS",
|
|
"cases": [{"alias": "CASE-1", "description": "Synthetic example. " * 500}],
|
|
"routing": {"allow_external": True, "allowed_external_providers": ["claude"]}}, ["claude"])
|
|
|
|
|
|
@pytest.mark.parametrize("subtype,code", [
|
|
("error_max_turns", "incomplete_generation"),
|
|
("error_max_structured_output_retries", "incomplete_generation"),
|
|
("error_max_budget_usd", "job_cost_budget_exhausted"),
|
|
("error_during_execution", "incomplete_generation"),
|
|
(CANARY, "incomplete_generation"),
|
|
])
|
|
def test_unsuccessful_final_is_specific_and_content_free(subtype, code):
|
|
raw = envelope(subtype=subtype, is_error=True, num_turns=4, structured_output=None)
|
|
with pytest.raises(Problem, match=code) as raised:
|
|
suite_backends.parse_claude(raw, MODEL, exit_code=1, subprocess_timeout_seconds=899.5)
|
|
details = raised.value.details
|
|
assert details["failure_stage"] == "cli_final_result"
|
|
assert details["exit_code"] == 1 and details["turns"] == 4
|
|
assert details["max_turns"] == CLAUDE_MAX_TURNS == 6
|
|
assert details["turn_limit_reached"] == (None if subtype == CANARY else subtype == "error_max_turns")
|
|
assert details["structured_retry_limit_reached"] == (None if subtype == CANARY else subtype == "error_max_structured_output_retries")
|
|
assert details["usage"] == {"input_tokens": 120, "output_tokens": 50}
|
|
assert details["structured_output_present"] is False
|
|
assert CANARY not in json.dumps(raised.value.document())
|
|
|
|
|
|
@pytest.mark.parametrize("status,code", [(429, "rate_limit"), (401, "provider_authentication"), (403, "provider_authentication")])
|
|
def test_provider_error_status_is_preserved(status, code):
|
|
with pytest.raises(Problem, match=code) as raised:
|
|
suite_backends.parse_claude(envelope(subtype="error_during_execution", is_error=True,
|
|
api_error_status=status), MODEL, exit_code=1)
|
|
assert raised.value.details["api_error_status"] == status
|
|
|
|
|
|
@pytest.mark.parametrize("reason", ["max_tokens", "model_context_window_exceeded"])
|
|
def test_explicit_provider_stop_distinct_from_turn_limit(reason):
|
|
with pytest.raises(Problem, match="incomplete_generation") as raised:
|
|
suite_backends.parse_claude(envelope(stop_reason=reason), MODEL, exit_code=0)
|
|
assert raised.value.details["failure_stage"] == "provider_stop"
|
|
assert raised.value.details["provider_stop_reason"] == reason
|
|
assert raised.value.details["turn_limit_reached"] is False
|
|
|
|
|
|
@pytest.mark.parametrize("raw,stage,code", [
|
|
("", "missing_init_or_final_event", "incomplete_generation"),
|
|
('{"type":"system","subtype":"init","model":"claude-opus-4-8"}', "missing_init_or_final_event", "incomplete_generation"),
|
|
("not json " + CANARY, "cli_event_json", "invalid_json_result"),
|
|
("[]", "cli_event_json", "invalid_json_result"),
|
|
])
|
|
def test_missing_final_and_malformed_stream_are_distinct(raw, stage, code):
|
|
with pytest.raises(Problem, match=code) as raised:
|
|
suite_backends.parse_claude(raw, MODEL, exit_code=0)
|
|
assert raised.value.details["failure_stage"] == stage
|
|
assert CANARY not in json.dumps(raised.value.document())
|
|
|
|
|
|
def test_text_or_tool_candidate_is_detected_but_not_silently_accepted():
|
|
value = json.loads(envelope().splitlines()[-1])["structured_output"]
|
|
assistant = {"type": "assistant", "message": {"stop_reason": "end_turn", "content": [
|
|
{"type": "tool_use", "name": "StructuredOutput", "input": value},
|
|
{"type": "text", "text": json.dumps(value)}]}}
|
|
raw = json.dumps(assistant) + "\n" + envelope(structured_output=None, result=json.dumps(value))
|
|
with pytest.raises(Problem, match="invalid_json_result") as raised:
|
|
suite_backends.parse_claude(raw, MODEL, exit_code=0)
|
|
details = raised.value.details
|
|
assert details["failure_stage"] == "structured_output_extraction"
|
|
assert details["assistant_structured_tool_input_present"] is True
|
|
assert details["assistant_json_text_present"] is True
|
|
assert details["final_json_text_present"] is True
|
|
assert CANARY not in json.dumps(details)
|
|
|
|
|
|
def test_valid_structured_result_and_exit_status_both_checked():
|
|
raw = envelope()
|
|
result, metadata = suite_backends.parse_claude(raw, MODEL, exit_code=0)
|
|
assert result["groups"][0]["description"] == CANARY
|
|
assert CANARY not in json.dumps(metadata)
|
|
for code in (1, -signal.SIGKILL):
|
|
with pytest.raises(Problem, match="incomplete_generation") as raised:
|
|
suite_backends.parse_claude(raw, MODEL, exit_code=code)
|
|
assert raised.value.details["failure_stage"] == "process_exit"
|
|
assert raised.value.details["structured_output_present"] is True
|
|
assert raised.value.details["termination_signal"] == (signal.SIGKILL if code < 0 else None)
|
|
|
|
|
|
def test_unknown_measurements_never_become_content_or_fabricated_zeroes():
|
|
details = snapshot(envelope(num_turns=CANARY, usage={"input_tokens": True, "output_tokens": float("nan")},
|
|
duration_api_ms=CANARY, total_cost_usd=float("inf"), stop_reason=CANARY))
|
|
assert details["turns"] is None and details["duration_api_ms"] is None
|
|
assert details["usage"] == {"input_tokens": None, "output_tokens": None}
|
|
assert details["provider_stop_reason"] == "other"
|
|
assert details["provider_timeout_seconds"] is None and details["reasoning_token_limit"] is None
|
|
assert CANARY not in json.dumps(details, allow_nan=False)
|
|
|
|
|
|
@pytest.mark.parametrize("mutation,code", [("schema", "invalid_json_result"), ("coverage", "invalid_case_assignments")])
|
|
def test_validation_failure_retains_usage_without_accepting_content(tmp_path, monkeypatch, capsys, mutation, code):
|
|
value = request()
|
|
selected = preflight(value)
|
|
result, metadata = suite_backends.parse_claude(envelope(), MODEL, exit_code=0)
|
|
if mutation == "schema":
|
|
result["groups"][0]["description"] = "x" * 241
|
|
else:
|
|
result["groups"][0]["members"].append("CASE-1")
|
|
monkeypatch.setattr(suite_backends, "switchyard_decision", lambda *_: None)
|
|
metadata["review_summary"] = {}
|
|
monkeypatch.setattr(suite_multipass, "generate", lambda *_: (result, metadata))
|
|
jobs = Jobs(tmp_path / "jobs.sqlite")
|
|
job, _ = jobs.submit("owner", "synthetic-validation", value, selected, "192.168.22.8", launch=False)
|
|
jobs.run(job["job_id"], "owner", value, selected, "192.168.22.8")
|
|
failed = jobs.get(job["job_id"], "owner")
|
|
assert failed["error"]["code"] == code
|
|
assert failed["error"]["details"]["failure_stage"] == "result_validation"
|
|
assert failed["usage"]["output_tokens"] == 50 and failed["turns"] == 2
|
|
assert not jobs.results
|
|
assert CANARY not in json.dumps(failed) + capsys.readouterr().out
|
|
|
|
|
|
@pytest.mark.parametrize("mode,code,stage", [
|
|
("success", None, None), ("nonzero", "incomplete_generation", "process_exit"),
|
|
("signal", "incomplete_generation", "process_exit"),
|
|
("empty", "backend_unavailable", "process_exit"),
|
|
("timeout", "timeout", "subprocess"), ("cancel", "cancelled", "subprocess"),
|
|
("oversize", "response_too_large", "subprocess"),
|
|
("start", "backend_unavailable", "process_start"),
|
|
])
|
|
def test_actual_subprocess_paths_cleanup_and_safe_metadata(tmp_path, monkeypatch, mode, code, stage):
|
|
real_tempdir = suite_backends.tempfile.TemporaryDirectory
|
|
real_read = Path.read_text
|
|
monkeypatch.setattr(suite_backends.tempfile, "TemporaryDirectory",
|
|
lambda **_: real_tempdir(prefix="suite-", dir=tmp_path))
|
|
monkeypatch.setattr(Path, "read_text", lambda p, *a, **k: "fake-token" if str(p) == "/vault/secrets/claude-token" else real_read(p, *a, **k))
|
|
script = "import os,signal,time; print(" + repr(envelope()) + ",flush=True); "
|
|
if mode in {"timeout", "cancel"}:
|
|
script += "time.sleep(30)"
|
|
elif mode == "signal":
|
|
script += "os.kill(os.getpid(),signal.SIGKILL)"
|
|
elif mode == "nonzero":
|
|
script += "raise SystemExit(1)"
|
|
elif mode == "empty":
|
|
script = "raise SystemExit(1)"
|
|
elif mode == "oversize":
|
|
monkeypatch.setattr(suite_backends, "OUTPUT_BYTES", 64)
|
|
elif mode == "success":
|
|
script += "time.sleep(0.25)"
|
|
command = ["/not-installed-synthetic-cli"] if mode == "start" else [sys.executable, "-c", script]
|
|
monkeypatch.setattr(suite_backends, "claude_command", lambda *_, **__: command)
|
|
value, cancel = request(), threading.Event()
|
|
if mode == "timeout":
|
|
value["execution"]["max_seconds"] = 0.02
|
|
if mode == "cancel":
|
|
cancel.set()
|
|
if code:
|
|
with pytest.raises(Problem, match=code) as raised:
|
|
suite_backends.claude_generate(value, cancel)
|
|
details = raised.value.details
|
|
assert details["failure_stage"] == stage
|
|
if mode in {"timeout", "cancel"}:
|
|
assert details["termination_signal"] == signal.SIGTERM
|
|
assert details["termination_reason"] == mode.replace("cancel", "cancelled")
|
|
assert CANARY not in json.dumps(details)
|
|
else:
|
|
updates = []
|
|
_, metadata = suite_backends.claude_generate(value, cancel, progress=updates.append)
|
|
assert metadata["cli_diagnostics"]["exit_code"] == 0
|
|
assert metadata["temporary_files_deleted"] is True
|
|
assert updates and updates[-1]["cli_output_bytes"] > 0
|
|
assert updates[-1]["cli_running"] is True
|
|
assert CANARY not in json.dumps(updates)
|
|
assert list(tmp_path.iterdir()) == []
|
|
|
|
|
|
def test_failed_cli_usage_survives_job_cleanup_without_content(tmp_path, monkeypatch, capsys):
|
|
value, jobs = request(), Jobs(tmp_path / "jobs.sqlite")
|
|
selected = preflight(value)
|
|
monkeypatch.setattr(suite_backends, "switchyard_decision", lambda *_: None)
|
|
def fail(*_, **kwargs):
|
|
suite_backends.parse_claude(envelope(subtype="error_max_turns", is_error=True,
|
|
structured_output=None), MODEL, exit_code=1)
|
|
monkeypatch.setattr(suite_backends, "claude_generate", fail)
|
|
document, _ = jobs.submit("owner", "synthetic-failure", value, selected, "192.168.22.8", launch=False)
|
|
jobs.run(document["job_id"], "owner", value, selected, "192.168.22.8")
|
|
failed = jobs.get(document["job_id"], "owner")
|
|
assert failed["error"]["details"]["turn_limit_reached"] is True
|
|
assert failed["usage"]["output_tokens"] == 50
|
|
assert not jobs.results and not jobs.active
|
|
replay, created = jobs.submit("owner", "synthetic-failure", value, selected, "192.168.22.8", launch=False)
|
|
assert not created and replay["job_id"] == document["job_id"]
|
|
assert CANARY not in json.dumps(failed) + capsys.readouterr().out
|
|
|
|
|
|
def test_process_exit_race_still_reaps(monkeypatch):
|
|
from types import SimpleNamespace
|
|
waited = []
|
|
def exited(*_):
|
|
raise ProcessLookupError()
|
|
monkeypatch.setattr(suite_backends.os, "killpg", exited)
|
|
process = SimpleNamespace(pid=1, poll=lambda: None, wait=lambda **_: waited.append(True))
|
|
suite_backends.stop(process)
|
|
assert waited == [True]
|