ariadne/ariadne/services/hermes_incident_issue.py
codex eebc9b16c4
All checks were successful
Tests / Declarative: Post Actions passed: 1404
feat(hermes): let an escalated diagnosis carry the fix it could not apply
Ariadne opens a pull request when it can: the repository is mapped, the file
is in the write allowlist, and the change is one anchored snippet the
validator can check. When any of that fails the incident escalates with a
diagnosis and nothing else - even though the model that wrote the diagnosis
frequently knows exactly what the fix is. That knowledge was discarded at the
moment it was most useful, because the cases where no patch is possible are
exactly the cases a maintainer has to do by hand.

A diagnosis may now carry up to three code suggestions: the file, what is
wrong there, and the code to change it to. Rendered into the issue under a
heading that says the change was not applied, because a code block in an issue
reads like something that already happened unless it is told otherwise.

Deliberately not a patch, and the difference is the safety story. A patch must
survive the validator because Ariadne acts on it. A suggestion is read by a
person who is already going to edit that file, so being wrong costs them a
moment's thought rather than a bad commit - which is why suggestions may
describe changes too large or too diffuse for the patcher to have attempted,
and why nothing here is anchored, applied or pushed.

Bounded at three. A diagnosis that suggests a dozen changes has stopped
diagnosing and started rewriting, and an issue that long buries the reason a
person was called in the first place.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-07 00:32:03 -03:00

433 lines
16 KiB
Python

"""File Gitea issues for triage incidents that concluded a human is needed.
When auto-triage decides a real service failure cannot be remediated under its
own authorization policy, the diagnosis is worth more in the failing service's
own issue tracker than in Ariadne's audit log. This module posts that
diagnosis as an issue in that repository so the operator finds it where they
already work.
The operation is additive and non-destructive: an issue is opened, nothing is
changed. Hermes never holds write access to these repositories — Ariadne posts
under its own automation token, and the issue body says so.
Filing is opt-in twice over: a master switch, and a per-job map of the
repositories that may receive issues. An unmapped job never causes a single
HTTP call.
"""
from __future__ import annotations
from typing import Any
import httpx
from ..utils.logging import get_logger
from . import hermes_incident_body as body
logger = get_logger(__name__)
ISSUE_EVENT_TYPE = "hermes_autotriage_issue"
HTTP_OK = 200
HTTP_CREATED = 201
DEDUPE_BY_CLASSIFICATION = "classification"
DEDUPE_BY_INCIDENT = "incident"
_DEFAULT_TIMEOUT_SECONDS = 15.0
_OPEN_ISSUES_LIMIT = 50
_DEFAULT_MAX_PER_TICK = 2
_FILED_KEY = "issues_filed"
_REPO_PAIR_LENGTH = 2
def find_open_incident_issue(cfg: dict, job: str, classification: str, incident_id: str) -> dict[str, Any]:
"""Find an already-open Hermes triage issue for this failure.
Inputs: `cfg` with gitea_base_url, gitea_token, owner, repo, and optional
dedupe_scope and timeout_seconds; the job, classification, and incident id
about to be filed. Outputs: {"found", "issue_number", "url", "error"},
returning the lowest-numbered match so repeated checks stay stable while
the issue sits open.
Dedupe scope "classification" (the default) treats any open issue carrying
the marker for the same job and classification as this failure's issue,
which is what stops a job that fails every build — a new incident each
time — from filing a new issue each time. Scope "incident" matches only
the same incident id.
Fails open by design: every HTTP, parse, or transport failure returns
found=False with `error` set. A false "already filed" would silently
swallow a real escalation and the operator would never learn the service
needs them, while a false "not filed yet" costs one duplicate issue.
Never raises and never logs the token.
"""
base_url = _base_url(cfg)
if not base_url:
return _no_issue("gitea base url is empty")
try:
with httpx.Client(timeout=_timeout(cfg)) as client:
response = client.get(
f"{base_url}/api/v1/repos/{_owner(cfg)}/{_repo(cfg)}/issues",
headers=_headers(cfg),
params={"state": "open", "limit": _OPEN_ISSUES_LIMIT},
)
except Exception as exc:
return _no_issue(f"open issue lookup failed: {exc}")
if response.status_code != HTTP_OK:
return _no_issue(f"open issue lookup http {response.status_code}")
return _lowest_matching_issue(response, _dedupe_scope(cfg), job, classification, incident_id)
def create_incident_issue(cfg: dict, context: dict) -> dict[str, Any]:
"""Open the triage issue that hands one incident to a human.
Inputs: `cfg` as for `find_open_incident_issue` plus optional
max_body_chars; `context` with incident_id, job, build_number, build_url,
classification, confidence, first_failed_gate, reason, facts, inferences,
authorize_reason, and run_id. Outputs: {"issue_number", "url", "error"}.
Only HTTP 201 counts as success. Never raises, never puts credentials in
the title or body, and never logs the token.
"""
base_url = _base_url(cfg)
if not base_url:
return {"issue_number": None, "url": None, "error": "gitea base url is empty"}
payload = {
"title": body.issue_title(context),
"body": body.issue_body(context, _max_body_chars(cfg)),
}
try:
with httpx.Client(timeout=_timeout(cfg)) as client:
response = client.post(
f"{base_url}/api/v1/repos/{_owner(cfg)}/{_repo(cfg)}/issues",
headers=_headers(cfg),
json=payload,
)
except Exception as exc:
return {"issue_number": None, "url": None, "error": f"issue create failed: {exc}"}
if response.status_code != HTTP_CREATED:
return {"issue_number": None, "url": None, "error": f"issue create http {response.status_code}"}
return _created_issue(response)
def maybe_file_issue(
storage: Any,
config: Any,
base: dict[str, Any],
diagnosis: dict[str, Any],
tick_state: dict[str, Any],
) -> dict[str, Any] | None:
"""File the incident issue when every opt-in gate allows it.
Inputs: the incident event storage; a settings-like object exposing the
hermes_issue_* and hermes_gitea_* values; the incident identity fields;
`diagnosis` with the evidence bundle, the parsed outcome (or None when the
run never completed), the authorization reason, and the Hermes run id; and
the tick-scoped state dict holding this tick's filed-issue counter.
Outputs: the recorded event detail, or None when nothing was attempted.
Files nothing unless the master switch is on, the job is mapped to a
repository, and this tick is still under its issue budget. Records one
bounded hermes_autotriage_issue event whenever a lookup ran, including the
deduped case. Never raises: issue filing sits on top of triage as a
courtesy and must never be able to break a tick.
"""
try:
return _file_issue(storage, config, base, diagnosis, tick_state)
except Exception as exc:
logger.info(
"hermes incident issue filing failed",
extra={
"event": "hermes_incident_issue",
"status": "error",
"incident_id": str(base.get("incident_id") or ""),
"detail": str(exc),
},
)
return None
def issue_config(config: Any) -> dict[str, Any]:
"""Build the issue-filing cfg dict from a settings-like object.
Inputs: an object exposing the hermes_issue_* and hermes_gitea_* settings.
Outputs: the cfg dict consumed by the lookup and create calls, reusing the
Gitea base URL and automation token already configured for code repair.
"""
return {
"enabled": bool(getattr(config, "hermes_issues_enabled", False)),
"repos": dict(getattr(config, "hermes_issue_repos", None) or {}),
"dedupe_scope": getattr(config, "hermes_issue_dedupe_scope", DEDUPE_BY_CLASSIFICATION),
"max_per_tick": getattr(config, "hermes_issue_max_per_tick", _DEFAULT_MAX_PER_TICK),
"gitea_base_url": getattr(config, "hermes_gitea_base_url", ""),
"gitea_token": getattr(config, "hermes_gitea_token", ""),
"timeout_seconds": _DEFAULT_TIMEOUT_SECONDS,
}
def issue_context(base: dict[str, Any], diagnosis: dict[str, Any]) -> dict[str, Any]:
"""Flatten the incident, evidence bundle, and outcome into body inputs.
Inputs: the incident identity fields and the diagnosis dict passed to
`maybe_file_issue`, optionally carrying the `code_proposal` detail the
orchestrator merged in when the same incident also opened a repair pull
request. Outputs: the context dict `create_incident_issue` renders. An
incident with no parsed decision still yields a context, with
classification "undiagnosed" so those failures dedupe together.
"""
decision = getattr(diagnosis.get("outcome"), "decision", None)
proposal = diagnosis.get("code_proposal")
bundle = diagnosis.get("bundle")
jenkins = bundle.get("jenkins") if isinstance(bundle, dict) else None
authorize_reason = str(diagnosis.get("authorize_reason") or "")
return {
"incident_id": str(base.get("incident_id") or ""),
"job": str(base.get("job") or ""),
"build_number": base.get("build_number"),
"build_url": str(jenkins.get("url") or "") if isinstance(jenkins, dict) else "",
"classification": str(
getattr(decision, "classification", "")
or diagnosis.get("classification")
or body.UNDIAGNOSED
),
"confidence": getattr(decision, "confidence", None),
"first_failed_gate": str(getattr(decision, "first_failed_gate", "") or ""),
"reason": str(getattr(decision, "reason", "") or authorize_reason),
"facts": [body.fact_fields(fact) for fact in getattr(decision, "facts", None) or []],
"inferences": list(getattr(decision, "inferences", None) or []),
"suggested_remediation": getattr(decision, "suggested_remediation", None),
"code_suggestions": getattr(decision, "code_suggestions", None) or [],
# The raw excerpt, so the issue shows the failure instead of citing
# where in Jenkins the failure can be found.
"bundle": bundle,
"authorize_reason": authorize_reason,
# An escalation that never reached a model has no cited facts, so its
# console text is the whole explanation and must not be dropped.
"observation": _observation(jenkins) if decision is None else "",
"run_id": diagnosis.get("run_id"),
"code_proposal_url": str(proposal.get("url") or "") if isinstance(proposal, dict) else "",
}
def _observation(jenkins: Any) -> str:
"""Return the console text Ariadne recorded without consulting a model."""
if not isinstance(jenkins, dict):
return ""
if jenkins.get("console_failures"):
return ""
return str(jenkins.get("console_tail") or "").strip()
def _file_issue(
storage: Any,
config: Any,
base: dict[str, Any],
diagnosis: dict[str, Any],
tick_state: dict[str, Any],
) -> dict[str, Any] | None:
"""Run the opt-in gates, the dedupe lookup, and the create."""
cfg = issue_config(config)
job = str(base.get("job") or "")
target = _repo_target(cfg["repos"].get(job))
if not cfg["enabled"] or target is None or _filed_count(tick_state) >= _max_per_tick(cfg):
return None
repo_cfg = {**cfg, "owner": target[0], "repo": target[1]}
context = issue_context(base, diagnosis)
existing = find_open_incident_issue(
repo_cfg, job, str(context["classification"]), str(context["incident_id"])
)
if existing.get("found"):
return _record(storage, context, existing, skipped=True)
tick_state[_FILED_KEY] = _filed_count(tick_state) + 1
return _record(storage, context, create_incident_issue(repo_cfg, context), skipped=False)
def _record(
storage: Any, context: dict[str, Any], result: dict[str, Any], *, skipped: bool
) -> dict[str, Any]:
"""Append the bounded issue event and return its detail."""
detail = {
"incident_id": context["incident_id"],
"job": context["job"],
"build_number": context["build_number"],
"classification": context["classification"],
"issue_number": result.get("issue_number"),
"url": result.get("url"),
"skipped": skipped,
"error": result.get("error"),
}
storage.record_event(ISSUE_EVENT_TYPE, detail)
logger.info(
"hermes incident issue handled",
extra={
"event": "hermes_incident_issue",
"status": "skipped" if skipped else "filed",
"incident_id": str(context["incident_id"]),
"detail": str(result.get("error") or ""),
},
)
return detail
def _lowest_matching_issue(
response: Any, scope: str, job: str, classification: str, incident_id: str
) -> dict[str, Any]:
"""Pick the lowest-numbered open issue matching the dedupe scope."""
try:
payload = response.json()
except Exception as exc:
return _no_issue(f"open issue parse failed: {exc}")
if not isinstance(payload, list):
return _no_issue("open issue payload is not a list")
matches = [
issue for issue in payload if _issue_matches(issue, scope, job, classification, incident_id)
]
if not matches:
return _no_issue(None)
lowest = min(matches, key=lambda issue: int(issue["number"]))
return {
"found": True,
"issue_number": int(lowest["number"]),
"url": str(lowest.get("html_url") or "") or None,
"error": None,
}
def _issue_matches(issue: Any, scope: str, job: str, classification: str, incident_id: str) -> bool:
"""Report whether one open issue is this incident's already-filed issue."""
if not isinstance(issue, dict):
return False
number = issue.get("number")
if isinstance(number, bool) or not isinstance(number, int):
return False
marker = body.parse_issue_marker(issue.get("body"))
if marker is None:
return False
if scope == DEDUPE_BY_INCIDENT:
return marker["incident"] == incident_id
return marker["job"] == job and marker["classification"] == classification
def _created_issue(response: Any) -> dict[str, Any]:
"""Map a 201 issue payload to the create result shape."""
try:
payload = response.json()
except Exception:
payload = {}
if not isinstance(payload, dict):
payload = {}
number = payload.get("number")
url = str(payload.get("html_url") or "") or None
if isinstance(number, bool) or not isinstance(number, int):
return {"issue_number": None, "url": url, "error": "issue create response had no number"}
return {"issue_number": number, "url": url, "error": None}
def _repo_target(value: Any) -> tuple[str, str] | None:
"""Resolve one issue-repo mapping entry to (owner, repo)."""
if isinstance(value, dict):
owner, repo = str(value.get("owner") or ""), str(value.get("repo") or "")
elif isinstance(value, (tuple, list)) and len(value) == _REPO_PAIR_LENGTH:
owner, repo = str(value[0]), str(value[1])
elif isinstance(value, str):
owner, _, repo = value.partition("/")
else:
return None
owner, repo = owner.strip(), repo.strip()
return (owner, repo) if owner and repo else None
def _no_issue(error: str | None) -> dict[str, Any]:
"""Build the fail-open result meaning "no open issue found"."""
return {"found": False, "issue_number": None, "url": None, "error": error}
def _dedupe_scope(cfg: dict) -> str:
"""Return the configured dedupe scope, defaulting to classification."""
scope = str(cfg.get("dedupe_scope") or "").strip().lower()
return DEDUPE_BY_INCIDENT if scope == DEDUPE_BY_INCIDENT else DEDUPE_BY_CLASSIFICATION
def _filed_count(tick_state: Any) -> int:
"""Return how many issues this tick has already filed."""
return _int_value(tick_state.get(_FILED_KEY)) if isinstance(tick_state, dict) else 0
def _max_per_tick(cfg: dict) -> int:
"""Return the per-tick issue budget, defaulting when unset or invalid."""
try:
return int(cfg.get("max_per_tick"))
except (TypeError, ValueError):
return _DEFAULT_MAX_PER_TICK
def _max_body_chars(cfg: dict) -> int:
"""Return the issue body character cap, defaulting when unset or invalid."""
try:
value = int(cfg.get("max_body_chars"))
except (TypeError, ValueError):
return body.DEFAULT_MAX_BODY_CHARS
return value if value > 0 else body.DEFAULT_MAX_BODY_CHARS
def _int_value(value: Any) -> int:
"""Coerce a value to int, defaulting to zero."""
try:
return int(value)
except (TypeError, ValueError):
return 0
def _headers(cfg: dict) -> dict[str, str]:
"""Build the Gitea token authorization header."""
return {"Authorization": f"token {cfg.get('gitea_token') or ''}"}
def _base_url(cfg: dict) -> str:
"""Return the configured Gitea base URL without a trailing slash."""
return str(cfg.get("gitea_base_url") or "").strip().rstrip("/")
def _owner(cfg: dict) -> str:
"""Return the configured repository owner."""
return str(cfg.get("owner") or "")
def _repo(cfg: dict) -> str:
"""Return the configured repository name."""
return str(cfg.get("repo") or "")
def _timeout(cfg: dict) -> float:
"""Return the bounded request timeout in seconds."""
try:
value = float(cfg.get("timeout_seconds"))
except (TypeError, ValueError):
return _DEFAULT_TIMEOUT_SECONDS
return value if value > 0 else _DEFAULT_TIMEOUT_SECONDS