from __future__ import annotations import json from types import SimpleNamespace from ariadne.services import hermes_autotriage as autotriage from ariadne.services import hermes_code_flow as module from ariadne.services.hermes_agent_client import HermesRunResult JOB = "hermes-code-demo" INCIDENT_ID = f"{JOB}/7" FILE_CONTENTS = "def discount(price):\n return price * 0.5\n" BUNDLE = {"incident_id": INCIDENT_ID, "jenkins": {"job": JOB}, "log_evidence": {"records": []}} class FakeStorage: def __init__(self) -> None: self.events: list[dict] = [] def record_event(self, event_type, detail) -> None: # type: ignore[no-untyped-def] self.events.append({"event_type": event_type, "detail": detail}) def list_events(self, limit=200, event_type=None): # type: ignore[no-untyped-def] rows = [ dict(row) for row in reversed(self.events) if event_type is None or row["event_type"] == event_type ] return rows[:limit] def _hermes_cfg() -> dict: return {"base_url": "http://hermes:8642", "api_key": "key", "total_timeout_seconds": 420.0} def _code_cfg(**overrides) -> dict: # type: ignore[no-untyped-def] base = { "candidate_path": "src/discount.py", "allowed_path_prefixes": ["src/"], "allowed_suffixes": [".py"], "max_patch_bytes": 4000, "max_changed_lines": 20, "gitea_base_url": "https://scm.example", "gitea_token": "secret-token", "owner": "bstein", "repo": "hermes-code-demo", "base_branch": "master", "timeout_seconds": 15.0, } base.update(overrides) return base def _model_output(**overrides) -> str: # type: ignore[no-untyped-def] payload = { "incident_id": INCIDENT_ID, "analysis": "The multiplier regressed to 0.5.", "patch": { "path": "src/discount.py", "original": "return price * 0.5", "replacement": "return price * 0.9", "rationale": "restore the intended discount", }, "human_required": False, "reason": "small localized fix", } payload.update(overrides) return json.dumps(payload) def _run(status: str = "completed", output=None) -> HermesRunResult: # type: ignore[no-untyped-def] return HermesRunResult( status=status, output=output, run_id="run-1", session_id="sess-1", error=None, duration_seconds=1.5, denied_approvals=0, ) def _install(monkeypatch, *, fetch=None, run=None, push=None, pull=None) -> dict: # type: ignore[no-untyped-def] calls: dict = {"fetches": [], "runs": [], "pushes": [], "pulls": []} def fake_fetch(cfg, path): # type: ignore[no-untyped-def] calls["fetches"].append((cfg, path)) return fetch if fetch is not None else (FILE_CONTENTS, None) def fake_run(cfg, prompt): # type: ignore[no-untyped-def] calls["runs"].append((cfg, prompt)) return run if run is not None else _run(output=_model_output()) def fake_push(cfg, incident_id, build_number, patch, contents): # type: ignore[no-untyped-def] calls["pushes"].append((cfg, incident_id, build_number, patch, contents)) return push if push is not None else {"branch": "hermes-repair/7", "committed": True, "error": None} def fake_pull(cfg, incident_id, build_number, branch, patch, analysis): # type: ignore[no-untyped-def] calls["pulls"].append((cfg, incident_id, build_number, branch, patch, analysis)) return pull if pull is not None else {"pr_number": 5, "url": "https://scm.example/pulls/5", "error": None} monkeypatch.setattr(module.hermes_code_repair, "fetch_file", fake_fetch) monkeypatch.setattr(module.hermes_agent_client, "run_triage", fake_run) monkeypatch.setattr(module.hermes_code_repair, "push_branch", fake_push) monkeypatch.setattr(module.hermes_code_repair, "open_pull_request", fake_pull) return calls def _propose(monkeypatch, **kwargs): # type: ignore[no-untyped-def] storage = FakeStorage() calls = _install(monkeypatch, **kwargs) result = module.propose_code_fix(storage, INCIDENT_ID, JOB, 7, BUNDLE, _hermes_cfg(), _code_cfg()) return storage, calls, result def _event(storage: FakeStorage) -> dict: rows = [row for row in storage.events if row["event_type"] == module.CODE_PROPOSAL_EVENT_TYPE] assert len(rows) == 1 return rows[0]["detail"] def test_happy_path_opens_pull_request(monkeypatch) -> None: storage, calls, result = _propose(monkeypatch) assert result == { "status": "pr_opened", "branch": "hermes-repair/7", "pr_number": 5, "url": "https://scm.example/pulls/5", "path": "src/discount.py", "run_id": "run-1", } assert calls["fetches"][0] == (_code_cfg(), "src/discount.py") cfg, incident_id, build_number, patch, contents = calls["pushes"][0] assert (incident_id, build_number) == (INCIDENT_ID, 7) assert patch.original == "return price * 0.5" assert contents == "def discount(price):\n return price * 0.9\n" _, _, _, branch, _, analysis = calls["pulls"][0] assert branch == "hermes-repair/7" assert analysis == "The multiplier regressed to 0.5." detail = _event(storage) assert detail == { "incident_id": INCIDENT_ID, "job": JOB, "build_number": 7, "run_id": "run-1", "validated": True, "reject_reason": None, "branch": "hermes-repair/7", "pr_number": 5, "url": "https://scm.example/pulls/5", } serialized = json.dumps(detail) assert "return price" not in serialized assert "secret-token" not in serialized def test_prompt_is_frozen_shape(monkeypatch) -> None: _, calls, _ = _propose(monkeypatch) cfg, prompt = calls["runs"][0] assert cfg == _hermes_cfg() assert prompt.startswith("Use $triage-titan-test-failures.\n") assert f"MINIMAL source fix for incident {INCIDENT_ID}" in prompt assert "The repository is bstein/hermes-code-demo branch master." in prompt assert '"incident_id": ""' in prompt assert "appearing exactly once" in prompt assert "Change as few lines as possible; do not reformat; do not add dependencies." in prompt assert "Ariadne validates and pushes the change — you do not execute anything." in prompt assert "Set human_required to true if the fix is not a small localized source change." in prompt assert json.dumps(BUNDLE, separators=(",", ":")) in prompt assert "Current content of the candidate file src/discount.py:" in prompt assert prompt.rstrip().endswith(FILE_CONTENTS.rstrip()) def test_fetch_failure_requires_human(monkeypatch) -> None: storage, calls, result = _propose(monkeypatch, fetch=(None, "file fetch http 404")) assert result == {"status": "human_required", "reason": "candidate_fetch_failed: file fetch http 404"} assert calls["runs"] == [] assert calls["pushes"] == [] detail = _event(storage) assert detail["run_id"] is None assert detail["validated"] is False assert detail["reject_reason"] == "candidate_fetch_failed: file fetch http 404" def test_unfinished_run_requires_human(monkeypatch) -> None: storage, calls, result = _propose(monkeypatch, run=_run(status="timeout")) assert result == {"status": "human_required", "reason": "hermes_run_timeout"} assert calls["pushes"] == [] assert _event(storage)["run_id"] == "run-1" def test_completed_run_without_output_requires_human(monkeypatch) -> None: _, calls, result = _propose(monkeypatch, run=_run(status="completed", output=None)) assert result == {"status": "human_required", "reason": "hermes_run_completed"} assert calls["pushes"] == [] def test_invalid_patch_response_requires_human(monkeypatch) -> None: storage, calls, result = _propose(monkeypatch, run=_run(output="no json here")) assert result == {"status": "human_required", "reason": "patch_invalid: no_json_object_found"} assert calls["pushes"] == [] assert _event(storage)["validated"] is False def test_patch_path_mismatch_requires_human(monkeypatch) -> None: output = _model_output( patch={ "path": "src/other.py", "original": "return price * 0.5", "replacement": "return price * 0.9", "rationale": "r", } ) _, calls, result = _propose(monkeypatch, run=_run(output=output)) assert result["reason"] == "patch_path_mismatch: got 'src/other.py' expected 'src/discount.py'" assert calls["pushes"] == [] def test_rejected_gate_requires_human(monkeypatch) -> None: output = _model_output( patch={ "path": "src/discount.py", "original": "not in the file", "replacement": "still not", "rationale": "r", } ) _, calls, result = _propose(monkeypatch, run=_run(output=output)) assert result == {"status": "human_required", "reason": "patch_rejected: original_missing"} assert calls["pushes"] == [] def test_push_failure_requires_human(monkeypatch) -> None: storage, calls, result = _propose( monkeypatch, push={"branch": "hermes-repair/7", "committed": False, "error": "commit http 500"} ) assert result == {"status": "human_required", "reason": "branch_push_failed: commit http 500"} assert calls["pulls"] == [] detail = _event(storage) assert detail["validated"] is True assert detail["branch"] is None def test_pull_request_failure_requires_human(monkeypatch) -> None: storage, _, result = _propose( monkeypatch, pull={"pr_number": None, "url": None, "error": "pull request http 500"} ) assert result == {"status": "human_required", "reason": "pull_request_failed: pull request http 500"} detail = _event(storage) assert detail["validated"] is True assert detail["branch"] == "hermes-repair/7" assert detail["pr_number"] is None def test_code_config_maps_settings() -> None: config = SimpleNamespace( hermes_code_candidate_path="src/discount.py", hermes_code_allowed_prefixes=["src/"], hermes_code_allowed_suffixes=[".py"], hermes_code_max_patch_bytes=4000, hermes_code_max_changed_lines=20, hermes_gitea_base_url="https://scm.example", hermes_gitea_token="secret-token", hermes_code_owner="bstein", hermes_code_repo="hermes-code-demo", hermes_code_base_branch="master", ) assert module.code_config(config) == _code_cfg() def _orchestrator_settings(**overrides) -> SimpleNamespace: # type: ignore[no-untyped-def] values = { "hermes_autotriage_enabled": True, "hermes_autotriage_job_allowlist": [JOB], "hermes_api_url": "http://hermes:8642", "hermes_api_key": "key", "hermes_run_timeout_seconds": 420.0, "hermes_code_enabled": True, "hermes_code_job": JOB, "hermes_code_owner": "bstein", "hermes_code_repo": "hermes-code-demo", "hermes_code_base_branch": "master", "hermes_code_candidate_path": "src/discount.py", "hermes_code_allowed_prefixes": ["src/"], "hermes_code_allowed_suffixes": [".py"], "hermes_code_max_patch_bytes": 4000, "hermes_code_max_changed_lines": 20, "hermes_gitea_base_url": "https://scm.example", "hermes_gitea_token": "secret-token", } values.update(overrides) return SimpleNamespace(**values) def _prepare_orchestrator(monkeypatch, flow_result, **setting_overrides): # type: ignore[no-untyped-def] storage = FakeStorage() calls: list = [] monkeypatch.setattr(autotriage, "settings", _orchestrator_settings(**setting_overrides)) monkeypatch.setattr( autotriage, "_fetch_last_build", lambda job: {"number": 7, "result": "FAILURE", "building": False, "url": "https://ci.example/7/"}, ) monkeypatch.setattr(autotriage.hermes_evidence, "collect_evidence", lambda i, j, b: dict(BUNDLE)) def fake_propose(storage_arg, incident_id, job, build_number, bundle, hermes_cfg, code_cfg): # type: ignore[no-untyped-def] calls.append((incident_id, job, build_number, bundle, hermes_cfg, code_cfg)) return flow_result monkeypatch.setattr(autotriage.hermes_code_flow, "propose_code_fix", fake_propose) triage_calls: list = [] def fake_run_triage(cfg, prompt): # type: ignore[no-untyped-def] triage_calls.append(prompt) return _run(status="error") monkeypatch.setattr(autotriage.hermes_agent_client, "run_triage", fake_run_triage) return storage, calls, triage_calls def test_orchestrator_records_code_fix_proposed_on_pr(monkeypatch) -> None: storage, calls, triage_calls = _prepare_orchestrator( monkeypatch, {"status": "pr_opened", "branch": "hermes-repair/7", "pr_number": 5, "url": "https://scm.example/pulls/5"}, ) summary = autotriage.run_hermes_autotriage(storage) assert summary["jobs"][JOB] == { "status": "human_required", "incident_id": INCIDENT_ID, "reason": "code_fix_proposed", } incidents = [row["detail"] for row in storage.events if row["event_type"] == autotriage.INCIDENT_EVENT_TYPE] assert [detail["status"] for detail in incidents] == ["detected", "human_required"] assert incidents[-1]["phase"] == { "reason": "code_fix_proposed", "branch": "hermes-repair/7", "pr_number": 5, "url": "https://scm.example/pulls/5", } assert [row for row in storage.events if row["event_type"] == autotriage.ACTION_EVENT_TYPE] == [] assert triage_calls == [] incident_id, job, build_number, bundle, hermes_cfg, code_cfg = calls[0] assert (incident_id, job, build_number) == (INCIDENT_ID, JOB, 7) assert bundle == BUNDLE assert hermes_cfg == _hermes_cfg() assert code_cfg == _code_cfg() def test_orchestrator_records_flow_failure_reason(monkeypatch) -> None: storage, _, _ = _prepare_orchestrator( monkeypatch, {"status": "human_required", "reason": "patch_rejected: path_unsafe"} ) summary = autotriage.run_hermes_autotriage(storage) assert summary["jobs"][JOB]["reason"] == "patch_rejected: path_unsafe" incidents = [row["detail"] for row in storage.events if row["event_type"] == autotriage.INCIDENT_EVENT_TYPE] assert incidents[-1]["status"] == "human_required" assert incidents[-1]["phase"] == {"reason": "patch_rejected: path_unsafe"} def test_orchestrator_defaults_missing_flow_reason(monkeypatch) -> None: storage, _, _ = _prepare_orchestrator(monkeypatch, {"status": "human_required"}) summary = autotriage.run_hermes_autotriage(storage) assert summary["jobs"][JOB]["reason"] == "code_fix_not_proposed" def test_orchestrator_skips_code_path_for_other_jobs(monkeypatch) -> None: storage, calls, triage_calls = _prepare_orchestrator( monkeypatch, {"status": "pr_opened"}, hermes_code_job="another-job" ) summary = autotriage.run_hermes_autotriage(storage) assert summary["jobs"][JOB]["status"] == "human_required" assert summary["jobs"][JOB]["reason"] == "hermes_run_error" assert calls == [] assert len(triage_calls) == 1