feat(hermes-triage): real services get both an issue and a patch proposal

The code path was gated on a single job id, so homegrown services could
never produce a pull request. Escalated incidents on mapped repositories
now additionally attempt a bounded patch proposal, and the filed issue
links it.

- hermes_code_flow.propose_for_incident: additive entry point invoked only
  from the escalation branch, so an auto-remediated failure never also
  gets a patch and a failed Hermes run never spends tokens on one
- three gates before any HTTP call: code enabled, not the legacy demo job,
  and the job has a repo mapping; unmapped jobs make zero calls
- never raises: a failed proposal cannot change the incident outcome or
  break the tick
- issue body links the proposal when a pull request was opened
- propose_code_fix added to the bounded action-label set

The legacy demo-job short-circuit is untouched and all of its tests pass
unchanged.

8 new tests; 480 pass in the hermes suite.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

ariadne/services/hermes_autotriage.py

@@ -18,6 +18,7 @@ from .hermes_autotriage_metrics import (
     HERMES_TRIAGE_DURATION_SECONDS,
     HERMES_TRIAGE_INCIDENT,
     HERMES_TRIAGE_LAST_SUCCESS_TS,
+    KNOWN_ACTION_LABELS,
     refresh_incident_gauges,
 )
 
@@ -30,7 +31,6 @@ ACTION_EVENT_TYPE = hermes_events.ACTION_EVENT_TYPE
 EXPECTED_CLASSIFICATION = "known_demo_fixture_failure"
 REPAIR_ACTION = "repair_demo_fixture"
 RETRY_ACTION = "retry_transient_infra"
-KNOWN_ACTION_IDS = (REPAIR_ACTION, RETRY_ACTION)
 REBUILD_FAILED_REASON = "repair rebuild failed"
 CODE_FIX_PROPOSED_REASON = "code_fix_proposed"
 
@@ -237,7 +237,13 @@ def _run_pipeline(  # noqa: PLR0913 - the tick's issue budget travels with the i
 def _authorize_and_execute(  # noqa: PLR0913 - the tick's issue budget travels with the incident context
     storage: Any, base: dict[str, Any], run: Any, outcome: Any, bundle: dict[str, Any], tick_state: dict[str, Any]
 ) -> dict[str, Any]:
-    """Gate the parsed diagnosis and run its action when every gate passes."""
+    """Gate the parsed diagnosis and run its action when every gate passes.
+
+    Escalating to a human is also where a mapped real service job earns a
+    code-fix proposal: the pull request rides along on the same human_required
+    phase and issue and never changes the outcome, while a job whose failure
+    was auto-remediated gets no patch at all.
+    """
 
     incident_id = str(base["incident_id"])
     matched, marker = _evidence_signature(outcome, bundle, incident_id)
@@ -256,8 +262,12 @@ def _authorize_and_execute(  # noqa: PLR0913 - the tick's issue budget travels w
         hermes_events.record_incident(storage, base, "diagnosed", hermes_events.outcome_phase(outcome))
     if not allowed:
         HERMES_TRIAGE_ACTION_TOTAL.labels(action=_action_label(outcome), result="rejected").inc()
-        hermes_events.record_incident(storage, base, "human_required", {"reason": reason})
+        proposal = hermes_code_flow.propose_for_incident(
-        _file_incident_issue(storage, base, _diagnosis(bundle, outcome, reason, run.run_id), tick_state)
+            storage, base, bundle, _hermes_run_config(), settings
+        )
+        hermes_events.record_incident(storage, base, "human_required", {"reason": reason, **proposal})
+        diagnosis = {**_diagnosis(bundle, outcome, reason, run.run_id), **proposal}
+        _file_incident_issue(storage, base, diagnosis, tick_state)
         return {"status": "human_required", "incident_id": incident_id, "reason": reason}
     return _execute_action(storage, base, outcome, marker)
 
@@ -435,7 +445,7 @@ def _action_label(outcome: Any) -> str:
     """
 
     action_id = _requested_action_id(outcome)
-    if action_id and (action_id in settings.hermes_allowed_actions or action_id in KNOWN_ACTION_IDS):
+    if action_id and (action_id in settings.hermes_allowed_actions or action_id in KNOWN_ACTION_LABELS):
         return action_id
     return _UNKNOWN_ACTION_LABEL
 

ariadne/services/hermes_autotriage_metrics.py

@@ -24,6 +24,15 @@ INCIDENT_STATUSES = (
     "failed",
 )
 
+PROPOSE_CODE_FIX_ACTION = "propose_code_fix"
+
+# The bounded `action` label set for HERMES_TRIAGE_ACTION_TOTAL. Anything the
+# model asks for that is neither deployed in hermes_allowed_actions nor listed
+# here collapses to "unknown", so a hallucinated action id can never mint a new
+# metric series. `propose_code_fix` is the orchestrator's own additive step
+# rather than a model-requested remediation, so it has to be named here.
+KNOWN_ACTION_LABELS = ("repair_demo_fixture", "retry_transient_infra", PROPOSE_CODE_FIX_ACTION)
+
 HERMES_TRIAGE_INCIDENT = Gauge(
     "ariadne_hermes_triage_incident",
     "Hermes auto-triage incident state (1=current status, 0=other statuses)",

ariadne/services/hermes_code_flow.py

@@ -7,6 +7,12 @@ console evidence says which files that build implicates (see
 publishes. The original single-repo demo settings still work unchanged - a
 job matching `hermes_code_job` keeps patching its one fixed
 `hermes_code_candidate_path` in `hermes_code_owner/hermes_code_repo`.
+
+Two entry points share that flow. `propose_code_fix` is the demo job's
+dedicated path, which replaces diagnosis outright. `propose_for_incident` is
+the additive step every other mapped job takes: it runs after triage has
+already diagnosed the failure and decided a human is needed, so a real service
+failure can produce both an issue and a pull request.
 """
 
 from __future__ import annotations
@@ -23,6 +29,7 @@ from . import (
     hermes_code_repair,
     hermes_code_repos,
 )
+from .hermes_autotriage_metrics import HERMES_TRIAGE_ACTION_TOTAL, PROPOSE_CODE_FIX_ACTION
 
 
 logger = get_logger(__name__)
@@ -127,6 +134,98 @@ def resolve_repo_config(job: str, settings_cfg: dict) -> dict[str, Any] | None:
     return hermes_code_repos.resolve(job, settings_cfg)
 
 
+def propose_for_incident(
+    storage: Any, base: dict, bundle: dict, hermes_cfg: dict, config: Any
+) -> dict[str, Any]:
+    """Attempt an extra code-fix proposal for an escalating incident.
+
+    Inputs: the incident event storage; the incident identity dict
+    (incident_id, job, build_number); the evidence bundle already collected
+    for that incident; the Hermes agent run config; and a settings-like object
+    exposing the hermes_code_* fields. Outputs: the phase detail to merge into
+    the incident's human_required phase - {"code_proposal": {branch, pr_number,
+    url}} when a pull request was opened, {"code_proposal": {"reason": ...}}
+    when the proposal declined, and {} when nothing was attempted.
+
+    Attempts nothing unless the code path is enabled, the job is a real
+    service job rather than the legacy demo job (which keeps its own dedicated
+    short-circuit), and the job maps to a repository - an unmapped job costs
+    no HTTP call and no model tokens. Additive on top of triage and never
+    raises: the incident's outcome is decided before this runs and must
+    survive any proposal failure.
+    """
+
+    try:
+        settings_cfg = _eligible_config(str(base.get("job") or ""), config)
+        if settings_cfg is None:
+            return {}
+        return _incident_proposal(storage, base, bundle, hermes_cfg, settings_cfg)
+    except Exception as exc:
+        logger.info(
+            "hermes incident code proposal failed",
+            extra={
+                "event": "hermes_code_flow",
+                "status": "error",
+                "incident_id": str(base.get("incident_id") or ""),
+                "detail": str(exc),
+            },
+        )
+        return {}
+
+
+def _eligible_config(job: str, config: Any) -> dict[str, Any] | None:
+    """Return the code cfg when this job may receive an extra proposal."""
+
+    if not job or not getattr(config, "hermes_code_enabled", False):
+        return None
+    if job == str(getattr(config, "hermes_code_job", "") or ""):
+        return None
+    settings_cfg = code_config(config)
+    return settings_cfg if resolve_repo_config(job, settings_cfg) is not None else None
+
+
+def _incident_proposal(
+    storage: Any, base: dict, bundle: dict, hermes_cfg: dict, settings_cfg: dict
+) -> dict[str, Any]:
+    """Run the proposal for an eligible incident and count its outcome."""
+
+    _count("requested")
+    result = propose_code_fix(
+        storage,
+        str(base.get("incident_id") or ""),
+        str(base.get("job") or ""),
+        _int_value(base.get("build_number")),
+        bundle,
+        hermes_cfg,
+        settings_cfg,
+    )
+    if result.get("status") != "pr_opened":
+        _count("rejected")
+        return {"code_proposal": {"reason": str(result.get("reason") or "code_fix_not_proposed")}}
+    _count("success")
+    detail = {
+        "branch": result.get("branch"),
+        "pr_number": result.get("pr_number"),
+        "url": result.get("url"),
+    }
+    return {"code_proposal": detail}
+
+
+def _count(result: str) -> None:
+    """Count one additive code-proposal attempt under its bounded label."""
+
+    HERMES_TRIAGE_ACTION_TOTAL.labels(action=PROPOSE_CODE_FIX_ACTION, result=result).inc()
+
+
+def _int_value(value: Any) -> int:
+    """Coerce a value to int, defaulting to zero."""
+
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
 def _propose(
     incident: _Incident, bundle: dict, hermes_cfg: dict, code_cfg: dict
 ) -> tuple[dict[str, Any], dict[str, Any]]:

ariadne/services/hermes_incident_body.py

@@ -176,16 +176,18 @@ def _inferences_section(context: dict) -> str:
 
 
 def _links_section(context: dict) -> str:
-    """Render the links pointing back at Jenkins and the Ariadne audit log."""
+    """Render the links pointing back at Jenkins, the proposal, and the audit log."""
 
     build_url = str(context.get("build_url") or "")
-    return "\n".join(
+    proposal_url = str(context.get("code_proposal_url") or "")
-        [
+    lines = [
-            "## Links",
+        "## Links",
-            f"- Failed build: {build_url}" if build_url else "- Failed build: url unavailable",
+        f"- Failed build: {build_url}" if build_url else "- Failed build: url unavailable",
-            f"- {_AUDIT_NOTE}",
+    ]
-        ]
+    if proposal_url:
-    )
+        lines.append(f"- Proposed fix awaiting review: {proposal_url}")
+    lines.append(f"- {_AUDIT_NOTE}")
+    return "\n".join(lines)
 
 
 def _bounded_body(body: str, budget: int) -> str:

ariadne/services/hermes_incident_issue.py

@@ -175,12 +175,15 @@ def issue_context(base: dict[str, Any], diagnosis: dict[str, Any]) -> dict[str,
     """Flatten the incident, evidence bundle, and outcome into body inputs.
 
     Inputs: the incident identity fields and the diagnosis dict passed to
-    `maybe_file_issue`. Outputs: the context dict `create_incident_issue`
+    `maybe_file_issue`, optionally carrying the `code_proposal` detail the
-    renders. An incident with no parsed decision still yields a context, with
+    orchestrator merged in when the same incident also opened a repair pull
+    request. Outputs: the context dict `create_incident_issue` renders. An
+    incident with no parsed decision still yields a context, with
     classification "undiagnosed" so those failures dedupe together.
     """
 
     decision = getattr(diagnosis.get("outcome"), "decision", None)
+    proposal = diagnosis.get("code_proposal")
     bundle = diagnosis.get("bundle")
     jenkins = bundle.get("jenkins") if isinstance(bundle, dict) else None
     authorize_reason = str(diagnosis.get("authorize_reason") or "")
@@ -197,6 +200,7 @@ def issue_context(base: dict[str, Any], diagnosis: dict[str, Any]) -> dict[str,
         "inferences": list(getattr(decision, "inferences", None) or []),
         "authorize_reason": authorize_reason,
         "run_id": diagnosis.get("run_id"),
+        "code_proposal_url": str(proposal.get("url") or "") if isinstance(proposal, dict) else "",
     }
 
 

tests/hermes_autotriage_harness.py

@@ -234,6 +234,57 @@ def _prepare(  # type: ignore[no-untyped-def]  # noqa: PLR0913
     return SimpleNamespace(storage=storage, calls=calls)
 
 
+def _code_settings(**overrides) -> SimpleNamespace:  # type: ignore[no-untyped-def]
+    values = {
+        "hermes_code_enabled": True,
+        "hermes_code_repos": f"{JOB}=bstein/{JOB}",
+        "hermes_issues_enabled": True,
+        "hermes_issue_repos": {JOB: ("bstein", JOB)},
+    }
+    values.update(overrides)
+    return _settings(**values)
+
+
+def _pr_opened(**overrides) -> dict:  # type: ignore[no-untyped-def]
+    payload = {
+        "status": "pr_opened",
+        "branch": "hermes-repair/12",
+        "pr_number": 8,
+        "url": "https://scm.example/pulls/8",
+    }
+    payload.update(overrides)
+    return payload
+
+
+def _prepare_code(monkeypatch, *, result=None, error=None, cfg=None, **kwargs):  # type: ignore[no-untyped-def]
+    env = _prepare(monkeypatch, cfg=cfg if cfg is not None else _code_settings(), **kwargs)
+    env.calls.update({"proposals": [], "gitea": [], "issues": []})
+
+    def fake_propose(storage, incident_id, job, build_number, bundle, hermes_cfg, code_cfg):  # type: ignore[no-untyped-def]  # noqa: PLR0913
+        env.calls["proposals"].append((incident_id, job, build_number, bundle, code_cfg))
+        if error is not None:
+            raise error
+        return result if result is not None else _pr_opened()
+
+    def fake_lookup(lookup_cfg):  # type: ignore[no-untyped-def]
+        env.calls["gitea"].append(lookup_cfg)
+        return {"found": False, "error": None}
+
+    def fake_create(issue_cfg, context):  # type: ignore[no-untyped-def]
+        env.calls["issues"].append(context)
+        return {"issue_number": 7, "url": "https://scm.example/issues/7", "error": None}
+
+    monkeypatch.setattr(module.hermes_code_flow, "propose_code_fix", fake_propose)
+    monkeypatch.setattr(module.hermes_code_flow.hermes_code_repair, "find_open_proposal", fake_lookup)
+    monkeypatch.setattr(
+        module.hermes_incident_issue,
+        "find_open_incident_issue",
+        lambda *args: {"found": False, "issue_number": None, "url": None, "error": None},
+    )
+    monkeypatch.setattr(module.hermes_incident_issue, "create_incident_issue", fake_create)
+    return env
+
+
 def _events(storage: FakeStorage, event_type: str) -> list[dict]:
     return [row["detail"] for row in storage.events if row["event_type"] == event_type]
 

tests/test_hermes_code_flow_repos.py

@@ -1,9 +1,10 @@
 """Multi-repository code-proposal tests for the Hermes code flow.
 
 The legacy single-repo demo path lives in test_hermes_code_flow.py; this file
-covers per-job repository resolution and evidence-driven candidate selection.
+covers per-job repository resolution, evidence-driven candidate selection, and
-Shared fakes are imported from that module so both drive the same storage,
+the additive proposal a real service job earns when triage escalates it.
-Gitea, and Hermes stand-ins.
+Shared fakes are imported from that module and from the auto-triage harness so
+every file drives the same storage, Gitea, and Hermes stand-ins.
 """
 
 from __future__ import annotations
@@ -11,7 +12,20 @@ from __future__ import annotations
 import json
 from types import SimpleNamespace
 
+import pytest
+
+from ariadne.services import hermes_autotriage as autotriage
 from ariadne.services import hermes_code_flow as module
+from tests.hermes_autotriage_harness import (
+    INCIDENT_ID as TRIAGE_INCIDENT_ID,
+    JOB as TRIAGE_JOB,
+    _code_settings,
+    _counter,
+    _events,
+    _model_output,
+    _prepare_code,
+    _statuses,
+)
 from tests.test_hermes_code_flow import (
     JOB,
     FakeStorage,
@@ -307,3 +321,109 @@ def test_validation_resolves_against_the_chosen_file(monkeypatch) -> None:
     assert contents == "def test_balance():\n    assert balance([1, 2]) == 2\n"
     assert _event(storage)["chosen_path"] == "tests/test_ledger.py"
 
+
+
+ESCALATION = _model_output(
+    classification="unknown_build_failure",
+    requested_action=None,
+    human_required=True,
+    reason="the failure matches no known signature",
+)
+
+
+def _last_phase(storage) -> dict:  # type: ignore[no-untyped-def]
+    return _events(storage, autotriage.INCIDENT_EVENT_TYPE)[-1]["phase"]
+
+
+def test_escalated_service_job_gets_both_an_issue_and_a_pull_request(monkeypatch) -> None:
+    requested = _counter("propose_code_fix", "requested")
+    success = _counter("propose_code_fix", "success")
+    env = _prepare_code(monkeypatch, run=_run(output=ESCALATION))
+
+    summary = autotriage.run_hermes_autotriage(env.storage)
+
+    assert summary["jobs"][TRIAGE_JOB] == {
+        "status": "human_required",
+        "incident_id": TRIAGE_INCIDENT_ID,
+        "reason": "human_required",
+    }
+    assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"]
+    incident_id, job, build_number, bundle, code_cfg = env.calls["proposals"][0]
+    assert (incident_id, job, build_number) == (TRIAGE_INCIDENT_ID, TRIAGE_JOB, 12)
+    assert bundle["incident_id"] == TRIAGE_INCIDENT_ID
+    assert code_cfg["repos"] == {TRIAGE_JOB: {"owner": "bstein", "repo": TRIAGE_JOB}}
+    assert _last_phase(env.storage) == {
+        "reason": "human_required",
+        "code_proposal": {
+            "branch": "hermes-repair/12",
+            "pr_number": 8,
+            "url": "https://scm.example/pulls/8",
+        },
+    }
+    assert env.calls["issues"][0]["code_proposal_url"] == "https://scm.example/pulls/8"
+    assert _counter("propose_code_fix", "requested") == requested + 1.0
+    assert _counter("propose_code_fix", "success") == success + 1.0
+
+
+def test_remediated_service_job_is_never_patched(monkeypatch) -> None:
+    env = _prepare_code(monkeypatch)
+
+    summary = autotriage.run_hermes_autotriage(env.storage)
+
+    assert summary["jobs"][TRIAGE_JOB]["status"] == "awaiting_rebuild"
+    assert env.calls["proposals"] == []
+    assert env.calls["gitea"] == []
+
+
+@pytest.mark.parametrize("overrides", [{"hermes_code_repos": ""}, {"hermes_code_enabled": False}])
+def test_out_of_scope_job_proposes_nothing_and_calls_no_gitea(monkeypatch, overrides) -> None:
+    env = _prepare_code(monkeypatch, cfg=_code_settings(**overrides), run=_run(output=ESCALATION))
+
+    summary = autotriage.run_hermes_autotriage(env.storage)
+
+    assert summary["jobs"][TRIAGE_JOB]["status"] == "human_required"
+    assert _last_phase(env.storage) == {"reason": "human_required"}
+    assert env.calls["proposals"] == []
+    assert env.calls["gitea"] == []
+
+
+def test_unfinished_hermes_run_proposes_nothing(monkeypatch) -> None:
+    env = _prepare_code(monkeypatch, run=_run(status="timeout"))
+
+    summary = autotriage.run_hermes_autotriage(env.storage)
+
+    assert summary["jobs"][TRIAGE_JOB]["reason"] == "hermes_run_timeout"
+    assert env.calls["proposals"] == []
+
+
+def test_declined_proposal_is_recorded_and_counted_rejected(monkeypatch) -> None:
+    rejected = _counter("propose_code_fix", "rejected")
+    env = _prepare_code(
+        monkeypatch,
+        result={"status": "human_required", "reason": "no_candidate_files"},
+        run=_run(output=ESCALATION),
+    )
+
+    summary = autotriage.run_hermes_autotriage(env.storage)
+
+    assert summary["jobs"][TRIAGE_JOB]["reason"] == "human_required"
+    assert _last_phase(env.storage) == {
+        "reason": "human_required",
+        "code_proposal": {"reason": "no_candidate_files"},
+    }
+    assert env.calls["issues"][0]["code_proposal_url"] == ""
+    assert _counter("propose_code_fix", "rejected") == rejected + 1.0
+
+
+def test_exploding_proposal_never_breaks_the_tick(monkeypatch) -> None:
+    env = _prepare_code(
+        monkeypatch, error=RuntimeError("gitea is unreachable"), run=_run(output=ESCALATION)
+    )
+
+    summary = autotriage.run_hermes_autotriage(env.storage)
+
+    assert summary["status"] == "ok"
+    assert summary["jobs"][TRIAGE_JOB]["status"] == "human_required"
+    assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"]
+    assert _last_phase(env.storage) == {"reason": "human_required"}
+    assert env.calls["issues"][0]["incident_id"] == TRIAGE_INCIDENT_ID

tests/test_hermes_incident_issue.py

@@ -10,6 +10,7 @@ JOB = "metis"
 INCIDENT_ID = f"{JOB}/12"
 CLASSIFICATION = "dependency_resolution_failure"
 TOKEN = "super-secret-token"
+PROPOSAL_URL = "https://scm.example/pulls/8"
 
 
 class FakeResponse:
@@ -346,6 +347,16 @@ def test_body_truncates_to_the_configured_cap_and_keeps_the_marker() -> None:
     assert rendered.rstrip().endswith(body.issue_marker(JOB, CLASSIFICATION, INCIDENT_ID))
 
 
+def test_a_proposed_fix_flows_from_the_diagnosis_into_the_issue_body() -> None:
+    proposal = {"branch": "hermes-repair/12", "pr_number": 8, "url": PROPOSAL_URL}
+    context = module.issue_context(_base(), _diagnosis(code_proposal=proposal))
+    assert context["code_proposal_url"] == PROPOSAL_URL
+    assert f"- Proposed fix awaiting review: {PROPOSAL_URL}" in body.issue_body(context)
+    assert module.issue_context(_base(), _diagnosis())["code_proposal_url"] == ""
+    assert module.issue_context(_base(), _diagnosis(code_proposal={}))["code_proposal_url"] == ""
+    assert "Proposed fix awaiting review" not in body.issue_body(_context())
+
+
 def test_context_defaults_to_undiagnosed_when_no_decision_was_parsed() -> None:
     context = module.issue_context(_base(), _diagnosis(outcome=None, run_id=None))