diff --git a/ariadne/services/hermes_autotriage.py b/ariadne/services/hermes_autotriage.py index 3726b31..213bf95 100644 --- a/ariadne/services/hermes_autotriage.py +++ b/ariadne/services/hermes_autotriage.py @@ -18,6 +18,7 @@ from .hermes_autotriage_metrics import ( HERMES_TRIAGE_DURATION_SECONDS, HERMES_TRIAGE_INCIDENT, HERMES_TRIAGE_LAST_SUCCESS_TS, + KNOWN_ACTION_LABELS, refresh_incident_gauges, ) @@ -30,7 +31,6 @@ ACTION_EVENT_TYPE = hermes_events.ACTION_EVENT_TYPE EXPECTED_CLASSIFICATION = "known_demo_fixture_failure" REPAIR_ACTION = "repair_demo_fixture" RETRY_ACTION = "retry_transient_infra" -KNOWN_ACTION_IDS = (REPAIR_ACTION, RETRY_ACTION) REBUILD_FAILED_REASON = "repair rebuild failed" CODE_FIX_PROPOSED_REASON = "code_fix_proposed" @@ -237,7 +237,13 @@ def _run_pipeline( # noqa: PLR0913 - the tick's issue budget travels with the i def _authorize_and_execute( # noqa: PLR0913 - the tick's issue budget travels with the incident context storage: Any, base: dict[str, Any], run: Any, outcome: Any, bundle: dict[str, Any], tick_state: dict[str, Any] ) -> dict[str, Any]: - """Gate the parsed diagnosis and run its action when every gate passes.""" + """Gate the parsed diagnosis and run its action when every gate passes. + + Escalating to a human is also where a mapped real service job earns a + code-fix proposal: the pull request rides along on the same human_required + phase and issue and never changes the outcome, while a job whose failure + was auto-remediated gets no patch at all. + """ incident_id = str(base["incident_id"]) matched, marker = _evidence_signature(outcome, bundle, incident_id) @@ -256,8 +262,12 @@ def _authorize_and_execute( # noqa: PLR0913 - the tick's issue budget travels w hermes_events.record_incident(storage, base, "diagnosed", hermes_events.outcome_phase(outcome)) if not allowed: HERMES_TRIAGE_ACTION_TOTAL.labels(action=_action_label(outcome), result="rejected").inc() - hermes_events.record_incident(storage, base, "human_required", {"reason": reason}) - _file_incident_issue(storage, base, _diagnosis(bundle, outcome, reason, run.run_id), tick_state) + proposal = hermes_code_flow.propose_for_incident( + storage, base, bundle, _hermes_run_config(), settings + ) + hermes_events.record_incident(storage, base, "human_required", {"reason": reason, **proposal}) + diagnosis = {**_diagnosis(bundle, outcome, reason, run.run_id), **proposal} + _file_incident_issue(storage, base, diagnosis, tick_state) return {"status": "human_required", "incident_id": incident_id, "reason": reason} return _execute_action(storage, base, outcome, marker) @@ -435,7 +445,7 @@ def _action_label(outcome: Any) -> str: """ action_id = _requested_action_id(outcome) - if action_id and (action_id in settings.hermes_allowed_actions or action_id in KNOWN_ACTION_IDS): + if action_id and (action_id in settings.hermes_allowed_actions or action_id in KNOWN_ACTION_LABELS): return action_id return _UNKNOWN_ACTION_LABEL diff --git a/ariadne/services/hermes_autotriage_metrics.py b/ariadne/services/hermes_autotriage_metrics.py index 7d561f0..c282cff 100644 --- a/ariadne/services/hermes_autotriage_metrics.py +++ b/ariadne/services/hermes_autotriage_metrics.py @@ -24,6 +24,15 @@ INCIDENT_STATUSES = ( "failed", ) +PROPOSE_CODE_FIX_ACTION = "propose_code_fix" + +# The bounded `action` label set for HERMES_TRIAGE_ACTION_TOTAL. Anything the +# model asks for that is neither deployed in hermes_allowed_actions nor listed +# here collapses to "unknown", so a hallucinated action id can never mint a new +# metric series. `propose_code_fix` is the orchestrator's own additive step +# rather than a model-requested remediation, so it has to be named here. +KNOWN_ACTION_LABELS = ("repair_demo_fixture", "retry_transient_infra", PROPOSE_CODE_FIX_ACTION) + HERMES_TRIAGE_INCIDENT = Gauge( "ariadne_hermes_triage_incident", "Hermes auto-triage incident state (1=current status, 0=other statuses)", diff --git a/ariadne/services/hermes_code_flow.py b/ariadne/services/hermes_code_flow.py index 87818a5..b77eee1 100644 --- a/ariadne/services/hermes_code_flow.py +++ b/ariadne/services/hermes_code_flow.py @@ -7,6 +7,12 @@ console evidence says which files that build implicates (see publishes. The original single-repo demo settings still work unchanged - a job matching `hermes_code_job` keeps patching its one fixed `hermes_code_candidate_path` in `hermes_code_owner/hermes_code_repo`. + +Two entry points share that flow. `propose_code_fix` is the demo job's +dedicated path, which replaces diagnosis outright. `propose_for_incident` is +the additive step every other mapped job takes: it runs after triage has +already diagnosed the failure and decided a human is needed, so a real service +failure can produce both an issue and a pull request. """ from __future__ import annotations @@ -23,6 +29,7 @@ from . import ( hermes_code_repair, hermes_code_repos, ) +from .hermes_autotriage_metrics import HERMES_TRIAGE_ACTION_TOTAL, PROPOSE_CODE_FIX_ACTION logger = get_logger(__name__) @@ -127,6 +134,98 @@ def resolve_repo_config(job: str, settings_cfg: dict) -> dict[str, Any] | None: return hermes_code_repos.resolve(job, settings_cfg) +def propose_for_incident( + storage: Any, base: dict, bundle: dict, hermes_cfg: dict, config: Any +) -> dict[str, Any]: + """Attempt an extra code-fix proposal for an escalating incident. + + Inputs: the incident event storage; the incident identity dict + (incident_id, job, build_number); the evidence bundle already collected + for that incident; the Hermes agent run config; and a settings-like object + exposing the hermes_code_* fields. Outputs: the phase detail to merge into + the incident's human_required phase - {"code_proposal": {branch, pr_number, + url}} when a pull request was opened, {"code_proposal": {"reason": ...}} + when the proposal declined, and {} when nothing was attempted. + + Attempts nothing unless the code path is enabled, the job is a real + service job rather than the legacy demo job (which keeps its own dedicated + short-circuit), and the job maps to a repository - an unmapped job costs + no HTTP call and no model tokens. Additive on top of triage and never + raises: the incident's outcome is decided before this runs and must + survive any proposal failure. + """ + + try: + settings_cfg = _eligible_config(str(base.get("job") or ""), config) + if settings_cfg is None: + return {} + return _incident_proposal(storage, base, bundle, hermes_cfg, settings_cfg) + except Exception as exc: + logger.info( + "hermes incident code proposal failed", + extra={ + "event": "hermes_code_flow", + "status": "error", + "incident_id": str(base.get("incident_id") or ""), + "detail": str(exc), + }, + ) + return {} + + +def _eligible_config(job: str, config: Any) -> dict[str, Any] | None: + """Return the code cfg when this job may receive an extra proposal.""" + + if not job or not getattr(config, "hermes_code_enabled", False): + return None + if job == str(getattr(config, "hermes_code_job", "") or ""): + return None + settings_cfg = code_config(config) + return settings_cfg if resolve_repo_config(job, settings_cfg) is not None else None + + +def _incident_proposal( + storage: Any, base: dict, bundle: dict, hermes_cfg: dict, settings_cfg: dict +) -> dict[str, Any]: + """Run the proposal for an eligible incident and count its outcome.""" + + _count("requested") + result = propose_code_fix( + storage, + str(base.get("incident_id") or ""), + str(base.get("job") or ""), + _int_value(base.get("build_number")), + bundle, + hermes_cfg, + settings_cfg, + ) + if result.get("status") != "pr_opened": + _count("rejected") + return {"code_proposal": {"reason": str(result.get("reason") or "code_fix_not_proposed")}} + _count("success") + detail = { + "branch": result.get("branch"), + "pr_number": result.get("pr_number"), + "url": result.get("url"), + } + return {"code_proposal": detail} + + +def _count(result: str) -> None: + """Count one additive code-proposal attempt under its bounded label.""" + + HERMES_TRIAGE_ACTION_TOTAL.labels(action=PROPOSE_CODE_FIX_ACTION, result=result).inc() + + +def _int_value(value: Any) -> int: + """Coerce a value to int, defaulting to zero.""" + + try: + return int(value) + except (TypeError, ValueError): + return 0 + + def _propose( incident: _Incident, bundle: dict, hermes_cfg: dict, code_cfg: dict ) -> tuple[dict[str, Any], dict[str, Any]]: diff --git a/ariadne/services/hermes_incident_body.py b/ariadne/services/hermes_incident_body.py index 00baa10..7f9df3a 100644 --- a/ariadne/services/hermes_incident_body.py +++ b/ariadne/services/hermes_incident_body.py @@ -176,16 +176,18 @@ def _inferences_section(context: dict) -> str: def _links_section(context: dict) -> str: - """Render the links pointing back at Jenkins and the Ariadne audit log.""" + """Render the links pointing back at Jenkins, the proposal, and the audit log.""" build_url = str(context.get("build_url") or "") - return "\n".join( - [ - "## Links", - f"- Failed build: {build_url}" if build_url else "- Failed build: url unavailable", - f"- {_AUDIT_NOTE}", - ] - ) + proposal_url = str(context.get("code_proposal_url") or "") + lines = [ + "## Links", + f"- Failed build: {build_url}" if build_url else "- Failed build: url unavailable", + ] + if proposal_url: + lines.append(f"- Proposed fix awaiting review: {proposal_url}") + lines.append(f"- {_AUDIT_NOTE}") + return "\n".join(lines) def _bounded_body(body: str, budget: int) -> str: diff --git a/ariadne/services/hermes_incident_issue.py b/ariadne/services/hermes_incident_issue.py index 4fbad81..3410f37 100644 --- a/ariadne/services/hermes_incident_issue.py +++ b/ariadne/services/hermes_incident_issue.py @@ -175,12 +175,15 @@ def issue_context(base: dict[str, Any], diagnosis: dict[str, Any]) -> dict[str, """Flatten the incident, evidence bundle, and outcome into body inputs. Inputs: the incident identity fields and the diagnosis dict passed to - `maybe_file_issue`. Outputs: the context dict `create_incident_issue` - renders. An incident with no parsed decision still yields a context, with + `maybe_file_issue`, optionally carrying the `code_proposal` detail the + orchestrator merged in when the same incident also opened a repair pull + request. Outputs: the context dict `create_incident_issue` renders. An + incident with no parsed decision still yields a context, with classification "undiagnosed" so those failures dedupe together. """ decision = getattr(diagnosis.get("outcome"), "decision", None) + proposal = diagnosis.get("code_proposal") bundle = diagnosis.get("bundle") jenkins = bundle.get("jenkins") if isinstance(bundle, dict) else None authorize_reason = str(diagnosis.get("authorize_reason") or "") @@ -197,6 +200,7 @@ def issue_context(base: dict[str, Any], diagnosis: dict[str, Any]) -> dict[str, "inferences": list(getattr(decision, "inferences", None) or []), "authorize_reason": authorize_reason, "run_id": diagnosis.get("run_id"), + "code_proposal_url": str(proposal.get("url") or "") if isinstance(proposal, dict) else "", } diff --git a/tests/hermes_autotriage_harness.py b/tests/hermes_autotriage_harness.py index 13c82e4..f7d55d7 100644 --- a/tests/hermes_autotriage_harness.py +++ b/tests/hermes_autotriage_harness.py @@ -234,6 +234,57 @@ def _prepare( # type: ignore[no-untyped-def] # noqa: PLR0913 return SimpleNamespace(storage=storage, calls=calls) +def _code_settings(**overrides) -> SimpleNamespace: # type: ignore[no-untyped-def] + values = { + "hermes_code_enabled": True, + "hermes_code_repos": f"{JOB}=bstein/{JOB}", + "hermes_issues_enabled": True, + "hermes_issue_repos": {JOB: ("bstein", JOB)}, + } + values.update(overrides) + return _settings(**values) + + +def _pr_opened(**overrides) -> dict: # type: ignore[no-untyped-def] + payload = { + "status": "pr_opened", + "branch": "hermes-repair/12", + "pr_number": 8, + "url": "https://scm.example/pulls/8", + } + payload.update(overrides) + return payload + + +def _prepare_code(monkeypatch, *, result=None, error=None, cfg=None, **kwargs): # type: ignore[no-untyped-def] + env = _prepare(monkeypatch, cfg=cfg if cfg is not None else _code_settings(), **kwargs) + env.calls.update({"proposals": [], "gitea": [], "issues": []}) + + def fake_propose(storage, incident_id, job, build_number, bundle, hermes_cfg, code_cfg): # type: ignore[no-untyped-def] # noqa: PLR0913 + env.calls["proposals"].append((incident_id, job, build_number, bundle, code_cfg)) + if error is not None: + raise error + return result if result is not None else _pr_opened() + + def fake_lookup(lookup_cfg): # type: ignore[no-untyped-def] + env.calls["gitea"].append(lookup_cfg) + return {"found": False, "error": None} + + def fake_create(issue_cfg, context): # type: ignore[no-untyped-def] + env.calls["issues"].append(context) + return {"issue_number": 7, "url": "https://scm.example/issues/7", "error": None} + + monkeypatch.setattr(module.hermes_code_flow, "propose_code_fix", fake_propose) + monkeypatch.setattr(module.hermes_code_flow.hermes_code_repair, "find_open_proposal", fake_lookup) + monkeypatch.setattr( + module.hermes_incident_issue, + "find_open_incident_issue", + lambda *args: {"found": False, "issue_number": None, "url": None, "error": None}, + ) + monkeypatch.setattr(module.hermes_incident_issue, "create_incident_issue", fake_create) + return env + + def _events(storage: FakeStorage, event_type: str) -> list[dict]: return [row["detail"] for row in storage.events if row["event_type"] == event_type] diff --git a/tests/test_hermes_code_flow_repos.py b/tests/test_hermes_code_flow_repos.py index 2dc91a5..e40dd09 100644 --- a/tests/test_hermes_code_flow_repos.py +++ b/tests/test_hermes_code_flow_repos.py @@ -1,9 +1,10 @@ """Multi-repository code-proposal tests for the Hermes code flow. The legacy single-repo demo path lives in test_hermes_code_flow.py; this file -covers per-job repository resolution and evidence-driven candidate selection. -Shared fakes are imported from that module so both drive the same storage, -Gitea, and Hermes stand-ins. +covers per-job repository resolution, evidence-driven candidate selection, and +the additive proposal a real service job earns when triage escalates it. +Shared fakes are imported from that module and from the auto-triage harness so +every file drives the same storage, Gitea, and Hermes stand-ins. """ from __future__ import annotations @@ -11,7 +12,20 @@ from __future__ import annotations import json from types import SimpleNamespace +import pytest + +from ariadne.services import hermes_autotriage as autotriage from ariadne.services import hermes_code_flow as module +from tests.hermes_autotriage_harness import ( + INCIDENT_ID as TRIAGE_INCIDENT_ID, + JOB as TRIAGE_JOB, + _code_settings, + _counter, + _events, + _model_output, + _prepare_code, + _statuses, +) from tests.test_hermes_code_flow import ( JOB, FakeStorage, @@ -307,3 +321,109 @@ def test_validation_resolves_against_the_chosen_file(monkeypatch) -> None: assert contents == "def test_balance():\n assert balance([1, 2]) == 2\n" assert _event(storage)["chosen_path"] == "tests/test_ledger.py" + + +ESCALATION = _model_output( + classification="unknown_build_failure", + requested_action=None, + human_required=True, + reason="the failure matches no known signature", +) + + +def _last_phase(storage) -> dict: # type: ignore[no-untyped-def] + return _events(storage, autotriage.INCIDENT_EVENT_TYPE)[-1]["phase"] + + +def test_escalated_service_job_gets_both_an_issue_and_a_pull_request(monkeypatch) -> None: + requested = _counter("propose_code_fix", "requested") + success = _counter("propose_code_fix", "success") + env = _prepare_code(monkeypatch, run=_run(output=ESCALATION)) + + summary = autotriage.run_hermes_autotriage(env.storage) + + assert summary["jobs"][TRIAGE_JOB] == { + "status": "human_required", + "incident_id": TRIAGE_INCIDENT_ID, + "reason": "human_required", + } + assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"] + incident_id, job, build_number, bundle, code_cfg = env.calls["proposals"][0] + assert (incident_id, job, build_number) == (TRIAGE_INCIDENT_ID, TRIAGE_JOB, 12) + assert bundle["incident_id"] == TRIAGE_INCIDENT_ID + assert code_cfg["repos"] == {TRIAGE_JOB: {"owner": "bstein", "repo": TRIAGE_JOB}} + assert _last_phase(env.storage) == { + "reason": "human_required", + "code_proposal": { + "branch": "hermes-repair/12", + "pr_number": 8, + "url": "https://scm.example/pulls/8", + }, + } + assert env.calls["issues"][0]["code_proposal_url"] == "https://scm.example/pulls/8" + assert _counter("propose_code_fix", "requested") == requested + 1.0 + assert _counter("propose_code_fix", "success") == success + 1.0 + + +def test_remediated_service_job_is_never_patched(monkeypatch) -> None: + env = _prepare_code(monkeypatch) + + summary = autotriage.run_hermes_autotriage(env.storage) + + assert summary["jobs"][TRIAGE_JOB]["status"] == "awaiting_rebuild" + assert env.calls["proposals"] == [] + assert env.calls["gitea"] == [] + + +@pytest.mark.parametrize("overrides", [{"hermes_code_repos": ""}, {"hermes_code_enabled": False}]) +def test_out_of_scope_job_proposes_nothing_and_calls_no_gitea(monkeypatch, overrides) -> None: + env = _prepare_code(monkeypatch, cfg=_code_settings(**overrides), run=_run(output=ESCALATION)) + + summary = autotriage.run_hermes_autotriage(env.storage) + + assert summary["jobs"][TRIAGE_JOB]["status"] == "human_required" + assert _last_phase(env.storage) == {"reason": "human_required"} + assert env.calls["proposals"] == [] + assert env.calls["gitea"] == [] + + +def test_unfinished_hermes_run_proposes_nothing(monkeypatch) -> None: + env = _prepare_code(monkeypatch, run=_run(status="timeout")) + + summary = autotriage.run_hermes_autotriage(env.storage) + + assert summary["jobs"][TRIAGE_JOB]["reason"] == "hermes_run_timeout" + assert env.calls["proposals"] == [] + + +def test_declined_proposal_is_recorded_and_counted_rejected(monkeypatch) -> None: + rejected = _counter("propose_code_fix", "rejected") + env = _prepare_code( + monkeypatch, + result={"status": "human_required", "reason": "no_candidate_files"}, + run=_run(output=ESCALATION), + ) + + summary = autotriage.run_hermes_autotriage(env.storage) + + assert summary["jobs"][TRIAGE_JOB]["reason"] == "human_required" + assert _last_phase(env.storage) == { + "reason": "human_required", + "code_proposal": {"reason": "no_candidate_files"}, + } + assert env.calls["issues"][0]["code_proposal_url"] == "" + assert _counter("propose_code_fix", "rejected") == rejected + 1.0 + + +def test_exploding_proposal_never_breaks_the_tick(monkeypatch) -> None: + env = _prepare_code( + monkeypatch, error=RuntimeError("gitea is unreachable"), run=_run(output=ESCALATION) + ) + + summary = autotriage.run_hermes_autotriage(env.storage) + + assert summary["status"] == "ok" + assert summary["jobs"][TRIAGE_JOB]["status"] == "human_required" + assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"] + assert _last_phase(env.storage) == {"reason": "human_required"} + assert env.calls["issues"][0]["incident_id"] == TRIAGE_INCIDENT_ID diff --git a/tests/test_hermes_incident_issue.py b/tests/test_hermes_incident_issue.py index e06eff2..70fd599 100644 --- a/tests/test_hermes_incident_issue.py +++ b/tests/test_hermes_incident_issue.py @@ -10,6 +10,7 @@ JOB = "metis" INCIDENT_ID = f"{JOB}/12" CLASSIFICATION = "dependency_resolution_failure" TOKEN = "super-secret-token" +PROPOSAL_URL = "https://scm.example/pulls/8" class FakeResponse: @@ -346,6 +347,16 @@ def test_body_truncates_to_the_configured_cap_and_keeps_the_marker() -> None: assert rendered.rstrip().endswith(body.issue_marker(JOB, CLASSIFICATION, INCIDENT_ID)) +def test_a_proposed_fix_flows_from_the_diagnosis_into_the_issue_body() -> None: + proposal = {"branch": "hermes-repair/12", "pr_number": 8, "url": PROPOSAL_URL} + context = module.issue_context(_base(), _diagnosis(code_proposal=proposal)) + assert context["code_proposal_url"] == PROPOSAL_URL + assert f"- Proposed fix awaiting review: {PROPOSAL_URL}" in body.issue_body(context) + assert module.issue_context(_base(), _diagnosis())["code_proposal_url"] == "" + assert module.issue_context(_base(), _diagnosis(code_proposal={}))["code_proposal_url"] == "" + assert "Proposed fix awaiting review" not in body.issue_body(_context()) + + def test_context_defaults_to_undiagnosed_when_no_decision_was_parsed() -> None: context = module.issue_context(_base(), _diagnosis(outcome=None, run_id=None))