ca5b22b977
All checks were successful
Tests / Declarative: Post Actions passed: 1377
The pull request said "Proposed by Hermes" and gave a reader nothing to check that against. "Hermes made this decision" is a claim; the run id is the receipt, and without it there is no way to open the run, read the prompt the model was given, or see the tool calls it made. The diagnosis issues have carried the run id from the start - the pull requests, which are the more consequential artifact, did not. The unused build_number parameter on open_pull_request is what the run id replaces. That parameter was dead - SonarQube flags it as python:S1172 on this very repository - so the argument count did not grow. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
494 lines
19 KiB
Python
494 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from types import SimpleNamespace
|
|
|
|
from ariadne.services import hermes_autotriage as autotriage
|
|
from ariadne.services import hermes_code_flow as module
|
|
from ariadne.services.hermes_agent_client import HermesRunResult
|
|
|
|
|
|
JOB = "hermes-code-demo"
|
|
INCIDENT_ID = f"{JOB}/7"
|
|
FILE_CONTENTS = "def discount(price):\n return price * 0.5\n"
|
|
BUNDLE = {"incident_id": INCIDENT_ID, "jenkins": {"job": JOB}, "log_evidence": {"records": []}}
|
|
|
|
|
|
class FakeStorage:
|
|
def __init__(self) -> None:
|
|
self.events: list[dict] = []
|
|
|
|
def record_event(self, event_type, detail) -> None: # type: ignore[no-untyped-def]
|
|
self.events.append({"event_type": event_type, "detail": detail})
|
|
|
|
def list_events(self, limit=200, event_type=None): # type: ignore[no-untyped-def]
|
|
rows = [
|
|
dict(row)
|
|
for row in reversed(self.events)
|
|
if event_type is None or row["event_type"] == event_type
|
|
]
|
|
return rows[:limit]
|
|
|
|
|
|
def _hermes_cfg() -> dict:
|
|
return {"base_url": "http://hermes:8642", "api_key": "key", "total_timeout_seconds": 420.0}
|
|
|
|
|
|
def _code_cfg(**overrides) -> dict: # type: ignore[no-untyped-def]
|
|
base = {
|
|
"hermes_ui_url": "",
|
|
"candidate_path": "src/discount.py",
|
|
"allowed_path_prefixes": ["src/"],
|
|
"allowed_suffixes": [".py"],
|
|
"max_patch_bytes": 4000,
|
|
"max_changed_lines": 20,
|
|
"gitea_base_url": "https://scm.example",
|
|
"gitea_token": "secret-token",
|
|
"owner": "bstein",
|
|
"repo": "hermes-code-demo",
|
|
"max_open_proposals": 0,
|
|
"fix_categories": [],
|
|
"base_branch": "master",
|
|
"timeout_seconds": 15.0,
|
|
"legacy_job": JOB,
|
|
"repos": {},
|
|
"job_prefixes": {},
|
|
"job_suffixes": {},
|
|
"job_base_branches": {},
|
|
"max_candidates": 3,
|
|
"max_context_chars": 60000,
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _model_output(**overrides) -> str: # type: ignore[no-untyped-def]
|
|
payload = {
|
|
"incident_id": INCIDENT_ID,
|
|
"analysis": "The multiplier regressed to 0.5.",
|
|
"patch": {
|
|
"path": "src/discount.py",
|
|
"original": "return price * 0.5",
|
|
"replacement": "return price * 0.9",
|
|
"rationale": "restore the intended discount",
|
|
},
|
|
"human_required": False,
|
|
"reason": "small localized fix",
|
|
}
|
|
payload.update(overrides)
|
|
return json.dumps(payload)
|
|
|
|
|
|
def _run(status: str = "completed", output=None) -> HermesRunResult: # type: ignore[no-untyped-def]
|
|
return HermesRunResult(
|
|
status=status,
|
|
output=output,
|
|
run_id="run-1",
|
|
session_id="sess-1",
|
|
error=None,
|
|
duration_seconds=1.5,
|
|
denied_approvals=0,
|
|
)
|
|
|
|
|
|
def _no_existing(**overrides) -> dict: # type: ignore[no-untyped-def]
|
|
base = {"found": False, "pr_number": None, "url": None, "branch": None, "error": None}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _install(monkeypatch, *, fetch=None, run=None, push=None, pull=None, existing=None) -> dict: # type: ignore[no-untyped-def]
|
|
calls: dict = {"fetches": [], "runs": [], "pushes": [], "pulls": [], "lookups": []}
|
|
|
|
def fake_find(cfg): # type: ignore[no-untyped-def]
|
|
calls["lookups"].append(cfg)
|
|
return existing if existing is not None else _no_existing()
|
|
|
|
monkeypatch.setattr(module.hermes_code_repair, "find_open_proposal", fake_find)
|
|
|
|
def fake_fetch(cfg, path): # type: ignore[no-untyped-def]
|
|
calls["fetches"].append((cfg, path))
|
|
if isinstance(fetch, dict):
|
|
return fetch.get(path, (None, "file fetch http 404"))
|
|
return fetch if fetch is not None else (FILE_CONTENTS, None)
|
|
|
|
def fake_run(cfg, prompt): # type: ignore[no-untyped-def]
|
|
calls["runs"].append((cfg, prompt))
|
|
return run if run is not None else _run(output=_model_output())
|
|
|
|
def fake_push(cfg, incident_id, build_number, patch, contents): # type: ignore[no-untyped-def]
|
|
calls["pushes"].append((cfg, incident_id, build_number, patch, contents))
|
|
return push if push is not None else {"branch": "hermes-repair/7", "committed": True, "error": None}
|
|
|
|
def fake_pull(cfg, incident_id, build_number, branch, patch, analysis): # type: ignore[no-untyped-def]
|
|
calls["pulls"].append((cfg, incident_id, build_number, branch, patch, analysis))
|
|
return pull if pull is not None else {"pr_number": 5, "url": "https://scm.example/pulls/5", "error": None}
|
|
|
|
monkeypatch.setattr(module.hermes_code_repair, "fetch_file", fake_fetch)
|
|
monkeypatch.setattr(module.hermes_agent_client, "run_triage", fake_run)
|
|
monkeypatch.setattr(module.hermes_code_repair, "push_branch", fake_push)
|
|
monkeypatch.setattr(module.hermes_code_repair, "open_pull_request", fake_pull)
|
|
return calls
|
|
|
|
|
|
def _propose(monkeypatch, code_cfg=None, **kwargs): # type: ignore[no-untyped-def]
|
|
storage = FakeStorage()
|
|
calls = _install(monkeypatch, **kwargs)
|
|
cfg = code_cfg if code_cfg is not None else _code_cfg()
|
|
result = module.propose_code_fix(storage, INCIDENT_ID, JOB, 7, BUNDLE, _hermes_cfg(), cfg)
|
|
return storage, calls, result
|
|
|
|
|
|
def _event(storage: FakeStorage) -> dict:
|
|
rows = [row for row in storage.events if row["event_type"] == module.CODE_PROPOSAL_EVENT_TYPE]
|
|
assert len(rows) == 1
|
|
return rows[0]["detail"]
|
|
|
|
|
|
def test_happy_path_opens_pull_request(monkeypatch) -> None:
|
|
storage, calls, result = _propose(monkeypatch)
|
|
assert result == {
|
|
"status": "pr_opened",
|
|
"branch": "hermes-repair/7",
|
|
"pr_number": 5,
|
|
"url": "https://scm.example/pulls/5",
|
|
"path": "src/discount.py",
|
|
"run_id": "run-1",
|
|
}
|
|
assert calls["fetches"][0] == (_code_cfg(), "src/discount.py")
|
|
cfg, incident_id, build_number, patch, contents = calls["pushes"][0]
|
|
assert (incident_id, build_number) == (INCIDENT_ID, 7)
|
|
assert patch.original == "return price * 0.5"
|
|
assert contents == "def discount(price):\n return price * 0.9\n"
|
|
_, _, _, branch, _, analysis = calls["pulls"][0]
|
|
assert branch == "hermes-repair/7"
|
|
assert analysis == "The multiplier regressed to 0.5."
|
|
detail = _event(storage)
|
|
assert detail == {
|
|
"incident_id": INCIDENT_ID,
|
|
"job": JOB,
|
|
"build_number": 7,
|
|
"run_id": "run-1",
|
|
"validated": True,
|
|
"reject_reason": None,
|
|
"branch": "hermes-repair/7",
|
|
"pr_number": 5,
|
|
"url": "https://scm.example/pulls/5",
|
|
"repo": "bstein/hermes-code-demo",
|
|
"candidates": ["src/discount.py"],
|
|
"chosen_path": "src/discount.py",
|
|
}
|
|
serialized = json.dumps(detail)
|
|
assert "return price" not in serialized
|
|
assert "secret-token" not in serialized
|
|
|
|
|
|
def test_existing_open_proposal_suppresses_duplicate(monkeypatch) -> None:
|
|
"""Only the ceiling suppresses new work, not a single open proposal."""
|
|
|
|
existing = {
|
|
"found": True,
|
|
"open_count": module.DEFAULT_MAX_OPEN_PROPOSALS,
|
|
"pr_number": 1,
|
|
"url": "https://scm.example/pulls/1",
|
|
"branch": "hermes-repair/4",
|
|
"error": None,
|
|
}
|
|
storage, calls, result = _propose(monkeypatch, existing=existing)
|
|
assert result == {
|
|
"status": "human_required",
|
|
"reason": "open_proposal_limit_reached",
|
|
"pr_number": 1,
|
|
"url": "https://scm.example/pulls/1",
|
|
"branch": "hermes-repair/4",
|
|
}
|
|
assert calls["lookups"] == [_code_cfg()]
|
|
assert calls["fetches"] == []
|
|
assert calls["runs"] == []
|
|
assert calls["pushes"] == []
|
|
assert calls["pulls"] == []
|
|
assert _event(storage) == {
|
|
"incident_id": INCIDENT_ID,
|
|
"job": JOB,
|
|
"build_number": 7,
|
|
"run_id": None,
|
|
"validated": False,
|
|
"reject_reason": "open_proposal_limit_reached",
|
|
"branch": "hermes-repair/4",
|
|
"pr_number": 1,
|
|
"url": "https://scm.example/pulls/1",
|
|
"repo": "bstein/hermes-code-demo",
|
|
"candidates": [],
|
|
"chosen_path": None,
|
|
}
|
|
|
|
|
|
def test_proposal_lookup_error_fails_open(monkeypatch) -> None:
|
|
storage, calls, result = _propose(
|
|
monkeypatch, existing=_no_existing(error="open proposal lookup http 503")
|
|
)
|
|
assert result["status"] == "pr_opened"
|
|
assert len(calls["fetches"]) == 1
|
|
assert len(calls["runs"]) == 1
|
|
assert _event(storage)["validated"] is True
|
|
|
|
|
|
def test_prompt_is_frozen_shape(monkeypatch) -> None:
|
|
_, calls, _ = _propose(monkeypatch)
|
|
cfg, prompt = calls["runs"][0]
|
|
assert cfg == _hermes_cfg()
|
|
assert prompt.startswith("Use $triage-titan-test-failures.\n")
|
|
assert f"MINIMAL source fix for incident {INCIDENT_ID}" in prompt
|
|
assert "The repository is bstein/hermes-code-demo branch master." in prompt
|
|
assert '"incident_id": "<must equal ' + INCIDENT_ID + '>"' in prompt
|
|
assert "appearing exactly once" in prompt
|
|
assert "Change as few lines as possible; do not reformat; do not add dependencies." in prompt
|
|
assert "Ariadne validates and pushes the change — you do not execute anything." in prompt
|
|
assert "Set human_required to true if the fix is not a small localized source change." in prompt
|
|
assert json.dumps(BUNDLE, separators=(",", ":")) in prompt
|
|
assert "Current content of the candidate file src/discount.py:" in prompt
|
|
assert prompt.rstrip().endswith(FILE_CONTENTS.rstrip())
|
|
|
|
|
|
def test_fetch_failure_requires_human(monkeypatch) -> None:
|
|
storage, calls, result = _propose(monkeypatch, fetch=(None, "file fetch http 404"))
|
|
assert result == {"status": "human_required", "reason": "candidate_fetch_failed: file fetch http 404"}
|
|
assert calls["runs"] == []
|
|
assert calls["pushes"] == []
|
|
detail = _event(storage)
|
|
assert detail["run_id"] is None
|
|
assert detail["validated"] is False
|
|
assert detail["reject_reason"] == "candidate_fetch_failed: file fetch http 404"
|
|
|
|
|
|
def test_unfinished_run_requires_human(monkeypatch) -> None:
|
|
storage, calls, result = _propose(monkeypatch, run=_run(status="timeout"))
|
|
assert result == {"status": "human_required", "reason": "hermes_run_timeout"}
|
|
assert calls["pushes"] == []
|
|
assert _event(storage)["run_id"] == "run-1"
|
|
|
|
|
|
def test_completed_run_without_output_requires_human(monkeypatch) -> None:
|
|
_, calls, result = _propose(monkeypatch, run=_run(status="completed", output=None))
|
|
assert result == {"status": "human_required", "reason": "hermes_run_completed"}
|
|
assert calls["pushes"] == []
|
|
|
|
|
|
def test_invalid_patch_response_requires_human(monkeypatch) -> None:
|
|
storage, calls, result = _propose(monkeypatch, run=_run(output="no json here"))
|
|
assert result == {"status": "human_required", "reason": "patch_invalid: no_json_object_found"}
|
|
assert calls["pushes"] == []
|
|
assert _event(storage)["validated"] is False
|
|
|
|
|
|
def test_patch_path_mismatch_requires_human(monkeypatch) -> None:
|
|
output = _model_output(
|
|
patch={
|
|
"path": "src/other.py",
|
|
"original": "return price * 0.5",
|
|
"replacement": "return price * 0.9",
|
|
"rationale": "r",
|
|
}
|
|
)
|
|
_, calls, result = _propose(monkeypatch, run=_run(output=output))
|
|
assert result["reason"] == "patch_path_mismatch: got 'src/other.py' expected 'src/discount.py'"
|
|
assert calls["pushes"] == []
|
|
|
|
|
|
def test_rejected_gate_requires_human(monkeypatch) -> None:
|
|
output = _model_output(
|
|
patch={
|
|
"path": "src/discount.py",
|
|
"original": "not in the file",
|
|
"replacement": "still not",
|
|
"rationale": "r",
|
|
}
|
|
)
|
|
_, calls, result = _propose(monkeypatch, run=_run(output=output))
|
|
assert result == {"status": "human_required", "reason": "patch_rejected: original_missing"}
|
|
assert calls["pushes"] == []
|
|
|
|
|
|
def test_push_failure_requires_human(monkeypatch) -> None:
|
|
storage, calls, result = _propose(
|
|
monkeypatch, push={"branch": "hermes-repair/7", "committed": False, "error": "commit http 500"}
|
|
)
|
|
assert result == {"status": "human_required", "reason": "branch_push_failed: commit http 500"}
|
|
assert calls["pulls"] == []
|
|
detail = _event(storage)
|
|
assert detail["validated"] is True
|
|
assert detail["branch"] is None
|
|
|
|
|
|
def test_pull_request_failure_requires_human(monkeypatch) -> None:
|
|
storage, _, result = _propose(
|
|
monkeypatch, pull={"pr_number": None, "url": None, "error": "pull request http 500"}
|
|
)
|
|
assert result == {"status": "human_required", "reason": "pull_request_failed: pull request http 500"}
|
|
detail = _event(storage)
|
|
assert detail["validated"] is True
|
|
assert detail["branch"] == "hermes-repair/7"
|
|
assert detail["pr_number"] is None
|
|
|
|
|
|
def test_code_config_maps_settings() -> None:
|
|
config = SimpleNamespace(
|
|
hermes_code_job=JOB,
|
|
hermes_code_candidate_path="src/discount.py",
|
|
hermes_code_allowed_prefixes=["src/"],
|
|
hermes_code_allowed_suffixes=[".py"],
|
|
hermes_code_max_patch_bytes=4000,
|
|
hermes_code_max_changed_lines=20,
|
|
hermes_gitea_base_url="https://scm.example",
|
|
hermes_gitea_token="secret-token",
|
|
hermes_code_owner="bstein",
|
|
hermes_code_repo="hermes-code-demo",
|
|
hermes_code_base_branch="master",
|
|
)
|
|
assert module.code_config(config) == _code_cfg()
|
|
|
|
def _orchestrator_settings(**overrides) -> SimpleNamespace: # type: ignore[no-untyped-def]
|
|
values = {
|
|
"hermes_autotriage_enabled": True,
|
|
"hermes_autotriage_job_allowlist": [JOB],
|
|
"hermes_api_url": "http://hermes:8642",
|
|
"hermes_api_key": "key",
|
|
"hermes_run_timeout_seconds": 420.0,
|
|
"hermes_code_enabled": True,
|
|
"hermes_code_job": JOB,
|
|
"hermes_code_owner": "bstein",
|
|
"hermes_code_repo": "hermes-code-demo",
|
|
"hermes_code_base_branch": "master",
|
|
"hermes_code_candidate_path": "src/discount.py",
|
|
"hermes_code_allowed_prefixes": ["src/"],
|
|
"hermes_code_allowed_suffixes": [".py"],
|
|
"hermes_code_max_patch_bytes": 4000,
|
|
"hermes_code_max_changed_lines": 20,
|
|
"hermes_gitea_base_url": "https://scm.example",
|
|
"hermes_gitea_token": "secret-token",
|
|
}
|
|
values.update(overrides)
|
|
return SimpleNamespace(**values)
|
|
|
|
|
|
def _prepare_orchestrator(monkeypatch, flow_result, **setting_overrides): # type: ignore[no-untyped-def]
|
|
storage = FakeStorage()
|
|
calls: list = []
|
|
monkeypatch.setattr(autotriage, "settings", _orchestrator_settings(**setting_overrides))
|
|
monkeypatch.setattr(
|
|
autotriage.hermes_jenkins_client,
|
|
"fetch_job_payload",
|
|
lambda job: {
|
|
"lastBuild": {
|
|
"number": 7,
|
|
"result": "FAILURE",
|
|
"building": False,
|
|
"url": "https://ci.example/7/",
|
|
}
|
|
},
|
|
)
|
|
monkeypatch.setattr(autotriage.hermes_evidence, "collect_evidence", lambda i, j, b: dict(BUNDLE))
|
|
|
|
def fake_propose(storage_arg, incident_id, job, build_number, bundle, hermes_cfg, code_cfg): # type: ignore[no-untyped-def]
|
|
calls.append((incident_id, job, build_number, bundle, hermes_cfg, code_cfg))
|
|
return flow_result
|
|
|
|
monkeypatch.setattr(autotriage.hermes_code_flow, "propose_code_fix", fake_propose)
|
|
triage_calls: list = []
|
|
|
|
def fake_run_triage(cfg, prompt): # type: ignore[no-untyped-def]
|
|
triage_calls.append(prompt)
|
|
return _run(status="error")
|
|
|
|
monkeypatch.setattr(autotriage.hermes_agent_client, "run_triage", fake_run_triage)
|
|
return storage, calls, triage_calls
|
|
|
|
|
|
def test_orchestrator_records_code_fix_proposed_on_pr(monkeypatch) -> None:
|
|
storage, calls, triage_calls = _prepare_orchestrator(
|
|
monkeypatch,
|
|
{"status": "pr_opened", "branch": "hermes-repair/7", "pr_number": 5, "url": "https://scm.example/pulls/5"},
|
|
)
|
|
summary = autotriage.run_hermes_autotriage(storage)
|
|
assert summary["jobs"][JOB] == {
|
|
"status": "human_required",
|
|
"incident_id": INCIDENT_ID,
|
|
"reason": "code_fix_proposed",
|
|
}
|
|
incidents = [row["detail"] for row in storage.events if row["event_type"] == autotriage.INCIDENT_EVENT_TYPE]
|
|
assert [detail["status"] for detail in incidents] == ["detected", "human_required"]
|
|
assert incidents[-1]["phase"] == {
|
|
"reason": "code_fix_proposed",
|
|
"branch": "hermes-repair/7",
|
|
"pr_number": 5,
|
|
"url": "https://scm.example/pulls/5",
|
|
}
|
|
assert [row for row in storage.events if row["event_type"] == autotriage.ACTION_EVENT_TYPE] == []
|
|
assert triage_calls == []
|
|
incident_id, job, build_number, bundle, hermes_cfg, code_cfg = calls[0]
|
|
assert (incident_id, job, build_number) == (INCIDENT_ID, JOB, 7)
|
|
assert bundle == BUNDLE
|
|
assert hermes_cfg == _hermes_cfg()
|
|
assert code_cfg == _code_cfg()
|
|
|
|
|
|
def test_orchestrator_records_flow_failure_reason(monkeypatch) -> None:
|
|
storage, _, _ = _prepare_orchestrator(
|
|
monkeypatch, {"status": "human_required", "reason": "patch_rejected: path_unsafe"}
|
|
)
|
|
summary = autotriage.run_hermes_autotriage(storage)
|
|
assert summary["jobs"][JOB]["reason"] == "patch_rejected: path_unsafe"
|
|
incidents = [row["detail"] for row in storage.events if row["event_type"] == autotriage.INCIDENT_EVENT_TYPE]
|
|
assert incidents[-1]["status"] == "human_required"
|
|
assert incidents[-1]["phase"] == {"reason": "patch_rejected: path_unsafe"}
|
|
|
|
|
|
def test_orchestrator_defaults_missing_flow_reason(monkeypatch) -> None:
|
|
storage, _, _ = _prepare_orchestrator(monkeypatch, {"status": "human_required"})
|
|
summary = autotriage.run_hermes_autotriage(storage)
|
|
assert summary["jobs"][JOB]["reason"] == "code_fix_not_proposed"
|
|
|
|
|
|
def test_orchestrator_skips_code_path_for_other_jobs(monkeypatch) -> None:
|
|
storage, calls, triage_calls = _prepare_orchestrator(
|
|
monkeypatch, {"status": "pr_opened"}, hermes_code_job="another-job"
|
|
)
|
|
summary = autotriage.run_hermes_autotriage(storage)
|
|
assert summary["jobs"][JOB]["status"] == "human_required"
|
|
assert summary["jobs"][JOB]["reason"] == "hermes_run_error"
|
|
assert calls == []
|
|
assert len(triage_calls) == 1
|
|
|
|
|
|
def test_open_proposals_below_the_ceiling_do_not_suppress_new_work(monkeypatch) -> None:
|
|
"""A fix for one failure must not be withheld because another awaits review."""
|
|
|
|
existing = {
|
|
"found": True,
|
|
"open_count": 3,
|
|
"pr_number": 1,
|
|
"url": "https://scm.example/pulls/1",
|
|
"branch": "hermes-repair/4",
|
|
"error": None,
|
|
}
|
|
_storage, _calls, result = _propose(monkeypatch, existing=existing)
|
|
|
|
assert result["status"] == "pr_opened"
|
|
|
|
|
|
def test_the_ceiling_is_configurable(monkeypatch) -> None:
|
|
"""Review capacity is a property of the reviewer, not of the code."""
|
|
|
|
existing = {
|
|
"found": True,
|
|
"open_count": 2,
|
|
"pr_number": 1,
|
|
"url": "https://scm.example/pulls/1",
|
|
"branch": "hermes-repair/4",
|
|
"error": None,
|
|
}
|
|
cfg = {**_code_cfg(), "max_open_proposals": 2}
|
|
_storage, _calls, result = _propose(monkeypatch, existing=existing, code_cfg=cfg)
|
|
|
|
assert result["reason"] == "open_proposal_limit_reached"
|