diff --git a/ariadne/services/hermes_autotriage.py b/ariadne/services/hermes_autotriage.py index 6d5881a..3f87a3f 100644 --- a/ariadne/services/hermes_autotriage.py +++ b/ariadne/services/hermes_autotriage.py @@ -41,6 +41,7 @@ _PROMPT_TEMPLATE = """Use $triage-titan-test-failures. Analyze incident __INCIDENT_ID__. Treat the attached Ariadne bundle as the source of truth. Identify the first enforced failure. +The jenkins.console_failures array holds excerpts around detected failure markers in chronological order; the earliest region usually contains the first enforced failure, and jenkins.console_tail is the end of the build which often only shows downstream noise. Distinguish facts from inference. Return ONLY a single JSON object with exactly these keys and no others: {"incident_id": "", "classification": "", "confidence": <0..1>, "facts": [{"statement": "...", "source": "jenkins|opensearch|victoriametrics|kubernetes|flux|gitea", "reference": "..."}], "inferences": ["..."], "first_failed_gate": "", "requested_action": {"type": "run_ariadne_job", "id": "repair_demo_fixture"} or null, "human_required": , "reason": ""} diff --git a/ariadne/services/hermes_autotriage_evidence.py b/ariadne/services/hermes_autotriage_evidence.py index 98b9ac9..674ace4 100644 --- a/ariadne/services/hermes_autotriage_evidence.py +++ b/ariadne/services/hermes_autotriage_evidence.py @@ -7,13 +7,15 @@ import httpx from ..settings import settings from .hermes_autotriage_logs import collect_log_evidence +from .hermes_console_evidence import extract_console_evidence SIGNATURE_MARKER = "hermes_demo_test_failure" SIGNATURE_TEST_NAME = "fixture-state-check" -_CONSOLE_TAIL_LINES = 100 -_CONSOLE_TAIL_MAX_BYTES = 8192 +_CONSOLE_MAX_CHARS = 2_000_000 +_CONSOLE_TAIL_KEEP_CHARS = 200_000 +_CONSOLE_TRUNCATION_NOTE = "... [ariadne] console truncated in the middle ..." _MAX_FAILED_TESTS = 10 _MAX_ERROR_DETAILS_CHARS = 2000 _FAILED_TEST_STATUSES = {"FAILED", "REGRESSION"} @@ -41,6 +43,8 @@ def collect_evidence(incident_id: str, job: str, last_build: dict) -> dict[str, "duration_seconds": _millis_to_seconds(last_build.get("duration")), "first_failed_stage": None, "console_tail": None, + "console_failures": [], + "console_truncated": False, "failed_tests": [], } build_number = jenkins["build_number"] @@ -48,7 +52,10 @@ def collect_evidence(incident_id: str, job: str, last_build: dict) -> dict[str, with httpx.Client(**_client_kwargs()) as client: jenkins["first_failed_stage"] = _first_failed_stage(client, job, build_number) jenkins["failed_tests"] = _failed_tests(client, job, build_number) - jenkins["console_tail"] = _console_tail(client, job, build_number) + console = _console_evidence(client, job, build_number) + jenkins["console_tail"] = console.get("tail") or None + jenkins["console_failures"] = console.get("regions") or [] + jenkins["console_truncated"] = bool(console.get("truncated")) except Exception: pass return { @@ -64,8 +71,8 @@ def evidence_has_signature(bundle: dict, incident_id: str) -> bool: Inputs: an evidence bundle from `collect_evidence` and the incident id. Outputs: True when the fixture-state-check test failed, the console tail - contains the demo failure marker, or the log records contain both the - marker and the incident id. + or any console failure region contains the demo failure marker, or the + log records contain both the marker and the incident id. """ jenkins = bundle.get("jenkins") if isinstance(bundle.get("jenkins"), dict) else {} @@ -73,7 +80,7 @@ def evidence_has_signature(bundle: dict, incident_id: str) -> bool: for test in failed_tests: if isinstance(test, dict) and test.get("name") == SIGNATURE_TEST_NAME: return True - if SIGNATURE_MARKER in str(jenkins.get("console_tail") or ""): + if SIGNATURE_MARKER in _console_signature_text(jenkins): return True log_evidence = bundle.get("log_evidence") if isinstance(bundle.get("log_evidence"), dict) else {} records = log_evidence.get("records") if isinstance(log_evidence.get("records"), list) else [] @@ -81,6 +88,17 @@ def evidence_has_signature(bundle: dict, incident_id: str) -> bool: return SIGNATURE_MARKER in messages and incident_id in messages +def _console_signature_text(jenkins: dict[str, Any]) -> str: + """Join the console tail and every failure region into one search string.""" + + regions = jenkins.get("console_failures") + parts = [str(jenkins.get("console_tail") or "")] + for region in regions if isinstance(regions, list) else []: + if isinstance(region, dict): + parts.append(str(region.get("text") or "")) + return "\n".join(parts) + + def _build_window(last_build: dict) -> tuple[str, str]: """Return the build start/end window as ISO-8601 UTC strings.""" @@ -154,17 +172,30 @@ def _failed_test(case: dict[str, Any]) -> dict[str, Any]: } -def _console_tail(client: httpx.Client, job: str, build_number: int) -> str | None: - """Return the byte-capped tail of the build console log, or None.""" +def _console_evidence(client: httpx.Client, job: str, build_number: int) -> dict[str, Any]: + """Return failure regions plus the tail of the build console log. + + The whole console is fetched (bounded by `_CONSOLE_MAX_CHARS`) because on + long pipelines the first enforced failure is thousands of lines above the + tail. Returns an empty dict when the console cannot be read. + """ try: response = client.get(f"{_base_url()}/job/{job}/{build_number}/consoleText") response.raise_for_status() text = response.text except Exception: - return None - tail = "\n".join(text.splitlines()[-_CONSOLE_TAIL_LINES:]) - return tail[-_CONSOLE_TAIL_MAX_BYTES:] + return {} + return extract_console_evidence(_bounded_console(text)) + + +def _bounded_console(text: str) -> str: + """Cap console text, keeping both the start and the end of huge logs.""" + + if len(text) <= _CONSOLE_MAX_CHARS: + return text + head = text[: _CONSOLE_MAX_CHARS - _CONSOLE_TAIL_KEEP_CHARS] + return f"{head}\n{_CONSOLE_TRUNCATION_NOTE}\n{text[-_CONSOLE_TAIL_KEEP_CHARS:]}" def _base_url() -> str: diff --git a/ariadne/services/hermes_code_flow.py b/ariadne/services/hermes_code_flow.py index 4dd60b4..a31b8dd 100644 --- a/ariadne/services/hermes_code_flow.py +++ b/ariadne/services/hermes_code_flow.py @@ -14,6 +14,7 @@ CODE_PROPOSAL_EVENT_TYPE = "hermes_autotriage_code_proposal" _RUN_COMPLETED = "completed" _GITEA_TIMEOUT_SECONDS = 15.0 +_EXISTING_PROPOSAL_REASON = "existing_proposal_open" _PATCH_PROMPT_TEMPLATE = """Use $triage-titan-test-failures. You are proposing a MINIMAL source fix for incident __INCIDENT_ID__. @@ -99,6 +100,9 @@ def _propose( ) -> tuple[dict[str, Any], dict[str, Any]]: """Run fetch, diagnosis, validation, and publication for one incident.""" + duplicate = _duplicate_proposal(code_cfg, incident_id) + if duplicate is not None: + return duplicate path = str(code_cfg.get("candidate_path") or "") contents, fetch_error = hermes_code_repair.fetch_file(code_cfg, path) if contents is None: @@ -120,6 +124,47 @@ def _propose( return _publish(code_cfg, incident_id, build_number, proposal) +def _duplicate_proposal( + code_cfg: dict, incident_id: str +) -> tuple[dict[str, Any], dict[str, Any]] | None: + """Suppress this proposal when an earlier repair pull request is still open. + + Runs before the candidate fetch and before Hermes, so a duplicate costs no + model tokens. Returns the human_required result/event pair when an open + proposal exists, otherwise None so the flow continues — including when the + lookup itself failed, which fails open rather than dropping real work. + """ + + existing = hermes_code_repair.find_open_proposal(code_cfg) + error = existing.get("error") + if error: + logger.info( + "hermes open proposal lookup failed", + extra={ + "event": "hermes_code_flow", + "status": "error", + "incident_id": incident_id, + "detail": error, + }, + ) + return None + if not existing.get("found"): + return None + identity = { + "pr_number": existing.get("pr_number"), + "url": existing.get("url"), + "branch": existing.get("branch"), + } + result = {"status": "human_required", "reason": _EXISTING_PROPOSAL_REASON, **identity} + event = { + "run_id": None, + "validated": False, + "reject_reason": _EXISTING_PROPOSAL_REASON, + **identity, + } + return result, event + + def _validated_patch( raw_output: str, incident_id: str, path: str, code_cfg: dict, contents: str ) -> tuple[hermes_code_patch.ProposedPatch | None, str]: diff --git a/ariadne/services/hermes_code_repair.py b/ariadne/services/hermes_code_repair.py index 5f53a07..260942c 100644 --- a/ariadne/services/hermes_code_repair.py +++ b/ariadne/services/hermes_code_repair.py @@ -18,6 +18,8 @@ HTTP_UNPROCESSABLE = 422 _DEFAULT_TIMEOUT_SECONDS = 15.0 _PROTECTED_BRANCHES = {"master", "main"} +_REPAIR_BRANCH_PREFIX = "hermes-repair/" +_OPEN_PULLS_LIMIT = 50 _COMMIT_IDENTITY = {"name": "Hermes Agent", "email": "hermes@bstein.dev"} _COMMIT_OK_STATUSES = {HTTP_OK, HTTP_CREATED} _BRANCH_RETRY_STATUSES = {HTTP_NOT_FOUND, HTTP_UNPROCESSABLE} @@ -46,6 +48,39 @@ def fetch_file(cfg: dict, path: str) -> tuple[str | None, str | None]: return response.text, None +def find_open_proposal(cfg: dict) -> dict[str, Any]: + """Find an already-open Hermes repair pull request for this repository. + + Inputs: `cfg` as for `fetch_file`. Outputs: + {"found", "pr_number", "url", "branch", "error"}. An open pull request + counts as an existing proposal when its head branch starts with + `hermes-repair/` and its base branch equals cfg["base_branch"]; the + lowest-numbered match (the original proposal) is returned so repeated + checks stay stable while the branch sits unmerged. + + Fails open by design: every HTTP, parse, or transport failure returns + found=False with `error` set, because a false "duplicate" would silently + suppress legitimate repair work — a missed duplicate is only noise, a + wrong duplicate is lost work. Never raises and never logs the token. + """ + + base_url = _base_url(cfg) + if not base_url: + return _no_proposal("gitea base url is empty") + try: + with httpx.Client(timeout=_timeout(cfg)) as client: + response = client.get( + f"{base_url}/api/v1/repos/{_owner(cfg)}/{_repo(cfg)}/pulls", + headers=_headers(cfg), + params={"state": "open", "limit": _OPEN_PULLS_LIMIT}, + ) + except Exception as exc: + return _no_proposal(f"open proposal lookup failed: {exc}") + if response.status_code != HTTP_OK: + return _no_proposal(f"open proposal lookup http {response.status_code}") + return _oldest_repair_pull(response, _base_branch(cfg)) + + def push_branch( cfg: dict, incident_id: str, build_number: int, patch: Any, patched_contents: str ) -> dict[str, Any]: @@ -162,6 +197,49 @@ def _file_sha(client: httpx.Client, cfg: dict, path: str) -> tuple[str | None, s return str(sha), None +def _oldest_repair_pull(response: Any, base_branch: str) -> dict[str, Any]: + """Pick the lowest-numbered repair proposal out of an open-pulls payload.""" + + try: + payload = response.json() + except Exception as exc: + return _no_proposal(f"open proposal parse failed: {exc}") + if not isinstance(payload, list): + return _no_proposal("open proposal payload is not a list") + matches = [pull for pull in payload if _is_repair_pull(pull, base_branch)] + if not matches: + return _no_proposal(None) + oldest = min(matches, key=lambda pull: int(pull["number"])) + return { + "found": True, + "pr_number": int(oldest["number"]), + "url": str(oldest.get("html_url") or "") or None, + "branch": str(oldest["head"].get("ref") or "") or None, + "error": None, + } + + +def _is_repair_pull(pull: Any, base_branch: str) -> bool: + """Report whether one open pull request is a Hermes repair proposal.""" + + if not isinstance(pull, dict): + return False + number = pull.get("number") + if isinstance(number, bool) or not isinstance(number, int): + return False + head, base = pull.get("head"), pull.get("base") + if not isinstance(head, dict) or not isinstance(base, dict): + return False + head_ref = str(head.get("ref") or "") + return head_ref.startswith(_REPAIR_BRANCH_PREFIX) and str(base.get("ref") or "") == base_branch + + +def _no_proposal(error: str | None) -> dict[str, Any]: + """Build the fail-open result meaning "no existing proposal found".""" + + return {"found": False, "pr_number": None, "url": None, "branch": None, "error": error} + + def _pr_result(response: Any) -> dict[str, Any]: """Map a pull-request response payload to the result shape.""" diff --git a/ariadne/services/hermes_console_evidence.py b/ariadne/services/hermes_console_evidence.py new file mode 100644 index 0000000..4087826 --- /dev/null +++ b/ariadne/services/hermes_console_evidence.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import re +from typing import Any + + +FAILURE_MARKERS: tuple[str, ...] = ( + # kubernetes / build-agent signals: long and unambiguous, so they are + # matched first and never mislabelled as a generic test failure + "ImagePullBackOff", + "OOMKilled", + "Failed to establish a new connection", + "Temporary failure in name resolution", + # pytest / unittest signals: they name the real failure + "=== FAILURES ===", + "short test summary", + "ERROR at setup", + "Traceback (most recent call last)", + "AssertionError", + "FAILED ", + "E ", + # generic build signals + "BUILD FAILED", + "FAILURE:", + "non-zero exit", + "exit code", + "command not found", + "No such file", + "Exception", + "ERROR:", + # tool gates + "Quality gate", + "fail-under", + "coverage", + "docstring", + "Semgrep", + "SonarQube", + "Trivy", + "ruff", + "[loc]", +) + +# Markers that are only meaningful at the start of a (stripped) line; matching +# them anywhere would swallow every line that merely says "... failed ...". +# All matching is case-insensitive, so these are stored lowercased. +PREFIX_ONLY_MARKERS: frozenset[str] = frozenset({"e ", "failed "}) + +_DEFAULTS: dict[str, int] = { + "context_before": 6, + "context_after": 12, + "tail_lines": 40, + "max_total_bytes": 12000, + "max_tail_bytes": 4000, + "max_regions": 6, +} + +_EMPTY: dict[str, Any] = {"regions": [], "tail": "", "total_lines": 0, "truncated": False} +_DIGITS = re.compile(r"\d+") + + +def extract_console_evidence(console_text: str, cfg: dict | None = None) -> dict: + """Extract failure regions and the tail from a Jenkins console log. + + Inputs: the full `consoleText` of a build and an optional config dict + (`context_before`, `context_after`, `tail_lines`, `max_total_bytes`, + `max_tail_bytes`, `max_regions`; each falls back to its default). + Outputs: {"regions": [{"marker", "line_number", "text"}, ...] in + chronological order, "tail": str, "total_lines": int, "truncated": bool}. + The tail alone is unreliable on long pipelines, so regions carry the + earliest enforced failure. Pure function, no I/O, never raises. + """ + + try: + options = _options(cfg) + lines = _lines(console_text) + regions, truncated = _budgeted(_deduped(_regions(lines, options)), options) + return { + "regions": regions, + "tail": _tail(lines, options), + "total_lines": len(lines), + "truncated": truncated, + } + except Exception: + return dict(_EMPTY) + + +def marker_for_line(line: str) -> str | None: + """Return the highest-priority failure marker matching one console line. + + Inputs: a single console line. Outputs: the marker as declared in + `FAILURE_MARKERS`, or None when the line carries no failure signal. + """ + + lowered = str(line).lower() + stripped = lowered.lstrip() + for marker in FAILURE_MARKERS: + needle = marker.lower() + if needle in PREFIX_ONLY_MARKERS: + if stripped.startswith(needle): + return marker + elif needle in lowered: + return marker + return None + + +def _lines(console_text: Any) -> list[str]: + """Split console text into lines, tolerating None and non-strings.""" + + return console_text.splitlines() if isinstance(console_text, str) else [] + + +def _options(cfg: dict | None) -> dict[str, int]: + """Merge caller config over the module defaults, ignoring bad values.""" + + source = cfg if isinstance(cfg, dict) else {} + return {key: _bounded_int(source.get(key), default) for key, default in _DEFAULTS.items()} + + +def _bounded_int(value: Any, default: int) -> int: + """Coerce a config value to a non-negative int, else return the default.""" + + try: + number = int(value) + except (TypeError, ValueError): + return default + return number if number >= 0 else default + + +def _regions(lines: list[str], options: dict[str, int]) -> list[dict[str, Any]]: + """Build merged context windows around every failure marker hit.""" + + spans: list[dict[str, Any]] = [] + for index, line in enumerate(lines): + marker = marker_for_line(line) + if marker is None: + continue + start = max(0, index - options["context_before"]) + end = min(len(lines), index + options["context_after"] + 1) + if spans and start <= spans[-1]["end"]: + spans[-1]["end"] = max(spans[-1]["end"], end) + continue + spans.append({"marker": marker, "line_number": index + 1, "start": start, "end": end}) + return [ + { + "marker": span["marker"], + "line_number": span["line_number"], + "text": "\n".join(lines[span["start"] : span["end"]]), + } + for span in spans + ] + + +def _deduped(regions: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Collapse near-identical repeated regions into one annotated region.""" + + first: dict[str, dict[str, Any]] = {} + counts: dict[str, int] = {} + order: list[str] = [] + for region in regions: + key = _fingerprint(region["text"]) + if key in first: + counts[key] += 1 + continue + first[key] = region + counts[key] = 1 + order.append(key) + return [_with_repeat_note(first[key], counts[key]) for key in order] + + +def _fingerprint(text: str) -> str: + """Normalize region text so retries differing only in numbers collapse.""" + + return _DIGITS.sub("#", " ".join(text.split())) + + +def _with_repeat_note(region: dict[str, Any], count: int) -> dict[str, Any]: + """Append a repeat note to a region seen more than once.""" + + if count <= 1: + return region + return {**region, "text": f"{region['text']}\n(repeated {count} times)"} + + +def _budgeted( + regions: list[dict[str, Any]], options: dict[str, int] +) -> tuple[list[dict[str, Any]], bool]: + """Keep the earliest regions that fit the byte and count budgets.""" + + limit = options["max_total_bytes"] + kept: list[dict[str, Any]] = [] + used = 0 + truncated = False + for region in regions: + if len(kept) >= options["max_regions"]: + return kept, True + size = _byte_len(region["text"]) + if used + size > limit: + if kept: + return kept, True + region = {**region, "text": _clip_head(region["text"], limit)} + size = _byte_len(region["text"]) + truncated = True + kept.append(region) + used += size + return kept, truncated + + +def _tail(lines: list[str], options: dict[str, int]) -> str: + """Return the byte-capped last lines of the console log.""" + + tail_lines = options["tail_lines"] + if tail_lines <= 0: + return "" + return _clip_tail("\n".join(lines[-tail_lines:]), options["max_tail_bytes"]) + + +def _byte_len(text: str) -> int: + """Return the UTF-8 byte length of a string.""" + + return len(text.encode("utf-8", "ignore")) + + +def _clip_head(text: str, limit: int) -> str: + """Return at most `limit` bytes from the start of the text.""" + + if limit <= 0: + return "" + return text.encode("utf-8", "ignore")[:limit].decode("utf-8", "ignore") + + +def _clip_tail(text: str, limit: int) -> str: + """Return at most `limit` bytes from the end of the text.""" + + if limit <= 0: + return "" + return text.encode("utf-8", "ignore")[-limit:].decode("utf-8", "ignore") diff --git a/tests/test_hermes_autotriage_evidence.py b/tests/test_hermes_autotriage_evidence.py index c4fc6d5..f5f15c6 100644 --- a/tests/test_hermes_autotriage_evidence.py +++ b/tests/test_hermes_autotriage_evidence.py @@ -138,6 +138,8 @@ def test_collect_evidence_full_bundle(monkeypatch) -> None: {"name": "flaky", "className": "demo.Flaky", "errorDetails": None}, ] assert jenkins["console_tail"] == "line1\nline2\nhermes_demo_test_failure seen" + assert jenkins["console_failures"] == [] + assert jenkins["console_truncated"] is False assert bundle["log_evidence"]["records"] == [] config, incident_id, window_start, window_end = calls["log"][0] assert incident_id == INCIDENT_ID @@ -179,14 +181,60 @@ def test_console_tail_caps_lines_and_bytes(monkeypatch) -> None: _install(monkeypatch, routes) tail = module.collect_evidence(INCIDENT_ID, JOB, _last_build())["jenkins"]["console_tail"] lines = tail.splitlines() - assert len(lines) == 100 - assert lines[0] == "line-200" + assert len(lines) == 40 + assert lines[0] == "line-260" assert lines[-1] == "line-299" routes["/consoleText"] = FakeResponse(text="x" * 20000) _install(monkeypatch, routes) tail = module.collect_evidence(INCIDENT_ID, JOB, _last_build())["jenkins"]["console_tail"] - assert len(tail) == 8192 + assert len(tail) == 4000 + + +def _long_console() -> str: + head = [ + "[Pipeline] stage: Unit Tests", + "+ python -m pytest tests -q", + "=================================== FAILURES ===================================", + "E assert 0 == 100", + "FAILED tests/test_ledger.py::test_balance", + ] + noise = [f"[Pipeline] teardown {i} completed ok" for i in range(2000)] + tail = [ + "java.io.IOException: Failed to archive artifacts", + "ERROR: script returned exit code 1", + "Finished: FAILURE", + ] + return "\n".join(head + noise + tail) + + +def test_console_failure_regions_capture_the_early_failure(monkeypatch) -> None: + routes = _all_failing_routes() + routes["/consoleText"] = FakeResponse(text=_long_console()) + _install(monkeypatch, routes) + jenkins = module.collect_evidence(INCIDENT_ID, JOB, _last_build())["jenkins"] + + regions = jenkins["console_failures"] + assert regions + assert "FAILED tests/test_ledger.py::test_balance" in regions[0]["text"] + assert regions[0]["line_number"] <= 5 + assert set(regions[0]) == {"marker", "line_number", "text"} + assert "FAILED tests/test_ledger.py::test_balance" not in jenkins["console_tail"] + assert "ERROR: script returned exit code 1" in jenkins["console_tail"] + assert jenkins["console_truncated"] is False + + +def test_console_read_is_bounded_but_keeps_head_and_tail(monkeypatch) -> None: + text = "\n".join( + ["ERROR: early boom"] + ["filler line that is long enough to add up" * 3] * 60000 + ["ERROR: late boom"] + ) + assert len(text) > module._CONSOLE_MAX_CHARS + routes = _all_failing_routes() + routes["/consoleText"] = FakeResponse(text=text) + _install(monkeypatch, routes) + jenkins = module.collect_evidence(INCIDENT_ID, JOB, _last_build())["jenkins"] + assert "ERROR: early boom" in jenkins["console_failures"][0]["text"] + assert "ERROR: late boom" in jenkins["console_tail"] def test_failed_tests_are_capped_and_error_details_truncated(monkeypatch) -> None: @@ -234,13 +282,28 @@ def test_evidence_tolerates_odd_jenkins_payloads(monkeypatch) -> None: assert jenkins["failed_tests"] == [{"name": "t", "className": "c", "errorDetails": None}] -def _bundle(failed_tests=(), console_tail="", records=()): # type: ignore[no-untyped-def] +def _bundle(failed_tests=(), console_tail="", records=(), console_failures=()): # type: ignore[no-untyped-def] return { - "jenkins": {"failed_tests": list(failed_tests), "console_tail": console_tail}, + "jenkins": { + "failed_tests": list(failed_tests), + "console_tail": console_tail, + "console_failures": list(console_failures), + }, "log_evidence": {"records": list(records)}, } +def test_signature_from_console_failure_region() -> None: + bundle = _bundle( + console_tail="ERROR: script returned exit code 1", + console_failures=[ + {"marker": "FAILED ", "line_number": 12, "text": "FAILED hermes_demo_test_failure check"}, + "not-a-region", + ], + ) + assert module.evidence_has_signature(bundle, INCIDENT_ID) is True + + def test_signature_from_failed_test_name() -> None: bundle = _bundle(failed_tests=[{"name": "fixture-state-check", "className": "c", "errorDetails": None}]) assert module.evidence_has_signature(bundle, INCIDENT_ID) is True diff --git a/tests/test_hermes_code_flow.py b/tests/test_hermes_code_flow.py index 80a8d6f..750c5fb 100644 --- a/tests/test_hermes_code_flow.py +++ b/tests/test_hermes_code_flow.py @@ -81,8 +81,20 @@ def _run(status: str = "completed", output=None) -> HermesRunResult: # type: ig ) -def _install(monkeypatch, *, fetch=None, run=None, push=None, pull=None) -> dict: # type: ignore[no-untyped-def] - calls: dict = {"fetches": [], "runs": [], "pushes": [], "pulls": []} +def _no_existing(**overrides) -> dict: # type: ignore[no-untyped-def] + base = {"found": False, "pr_number": None, "url": None, "branch": None, "error": None} + base.update(overrides) + return base + + +def _install(monkeypatch, *, fetch=None, run=None, push=None, pull=None, existing=None) -> dict: # type: ignore[no-untyped-def] + calls: dict = {"fetches": [], "runs": [], "pushes": [], "pulls": [], "lookups": []} + + def fake_find(cfg): # type: ignore[no-untyped-def] + calls["lookups"].append(cfg) + return existing if existing is not None else _no_existing() + + monkeypatch.setattr(module.hermes_code_repair, "find_open_proposal", fake_find) def fake_fetch(cfg, path): # type: ignore[no-untyped-def] calls["fetches"].append((cfg, path)) @@ -155,6 +167,50 @@ def test_happy_path_opens_pull_request(monkeypatch) -> None: assert "secret-token" not in serialized +def test_existing_open_proposal_suppresses_duplicate(monkeypatch) -> None: + existing = { + "found": True, + "pr_number": 1, + "url": "https://scm.example/pulls/1", + "branch": "hermes-repair/4", + "error": None, + } + storage, calls, result = _propose(monkeypatch, existing=existing) + assert result == { + "status": "human_required", + "reason": "existing_proposal_open", + "pr_number": 1, + "url": "https://scm.example/pulls/1", + "branch": "hermes-repair/4", + } + assert calls["lookups"] == [_code_cfg()] + assert calls["fetches"] == [] + assert calls["runs"] == [] + assert calls["pushes"] == [] + assert calls["pulls"] == [] + assert _event(storage) == { + "incident_id": INCIDENT_ID, + "job": JOB, + "build_number": 7, + "run_id": None, + "validated": False, + "reject_reason": "existing_proposal_open", + "branch": "hermes-repair/4", + "pr_number": 1, + "url": "https://scm.example/pulls/1", + } + + +def test_proposal_lookup_error_fails_open(monkeypatch) -> None: + storage, calls, result = _propose( + monkeypatch, existing=_no_existing(error="open proposal lookup http 503") + ) + assert result["status"] == "pr_opened" + assert len(calls["fetches"]) == 1 + assert len(calls["runs"]) == 1 + assert _event(storage)["validated"] is True + + def test_prompt_is_frozen_shape(monkeypatch) -> None: _, calls, _ = _propose(monkeypatch) cfg, prompt = calls["runs"][0] diff --git a/tests/test_hermes_code_repair.py b/tests/test_hermes_code_repair.py index 8c74bc4..84f20ca 100644 --- a/tests/test_hermes_code_repair.py +++ b/tests/test_hermes_code_repair.py @@ -123,6 +123,96 @@ def test_fetch_file_uses_default_timeout(monkeypatch) -> None: assert calls["kwargs"] == {"timeout": 15.0} +def _pull(number: int, head: str = BRANCH, base: str = "master") -> dict: + return { + "number": number, + "html_url": f"https://scm.example/pulls/{number}", + "head": {"ref": head}, + "base": {"ref": base}, + } + + +def _none_found(error=None) -> dict: # type: ignore[no-untyped-def] + return {"found": False, "pr_number": None, "url": None, "branch": None, "error": error} + + +def test_find_open_proposal_matches_repair_branch(monkeypatch) -> None: + calls = _install_http(monkeypatch, [FakeResponse(200, [_pull(2)])]) + result = module.find_open_proposal(_cfg()) + assert result == { + "found": True, + "pr_number": 2, + "url": "https://scm.example/pulls/2", + "branch": BRANCH, + "error": None, + } + method, url, kwargs = calls["requests"][0] + assert (method, url) == ("GET", "https://scm.example/api/v1/repos/bstein/hermes-code-demo/pulls") + assert kwargs["params"] == {"state": "open", "limit": 50} + assert kwargs["headers"] == {"Authorization": "token secret-token"} + + +def test_find_open_proposal_ignores_other_branch_prefixes(monkeypatch) -> None: + _install_http(monkeypatch, [FakeResponse(200, [_pull(3, head="feature/thing"), _pull(4, head="renovate/x")])]) + assert module.find_open_proposal(_cfg()) == _none_found() + + +def test_find_open_proposal_ignores_wrong_base_branch(monkeypatch) -> None: + _install_http(monkeypatch, [FakeResponse(200, [_pull(3, base="develop")])]) + assert module.find_open_proposal(_cfg()) == _none_found() + + +def test_find_open_proposal_returns_lowest_numbered_match(monkeypatch) -> None: + payload = [ + _pull(9, head="hermes-repair/9"), + _pull(2, head="hermes-repair/4"), + _pull(1, head="other/1"), + _pull(5, head="hermes-repair/5"), + ] + _install_http(monkeypatch, [FakeResponse(200, payload)]) + result = module.find_open_proposal(_cfg()) + assert (result["found"], result["pr_number"], result["branch"]) == (True, 2, "hermes-repair/4") + + +def test_find_open_proposal_skips_malformed_entries(monkeypatch) -> None: + payload = ["not a dict", {"number": 1}, {"number": True, "head": {"ref": BRANCH}, "base": {"ref": "master"}}] + _install_http(monkeypatch, [FakeResponse(200, payload)]) + assert module.find_open_proposal(_cfg()) == _none_found() + + +def test_find_open_proposal_empty_list(monkeypatch) -> None: + _install_http(monkeypatch, [FakeResponse(200, [])]) + assert module.find_open_proposal(_cfg()) == _none_found() + + +def test_find_open_proposal_http_error_fails_open(monkeypatch) -> None: + _install_http(monkeypatch, [FakeResponse(503)]) + assert module.find_open_proposal(_cfg()) == _none_found("open proposal lookup http 503") + + +def test_find_open_proposal_malformed_json_fails_open(monkeypatch) -> None: + _install_http(monkeypatch, [FakeResponse(200)]) + result = module.find_open_proposal(_cfg()) + assert result["found"] is False + assert result["error"] == "open proposal parse failed: no json body" + + +def test_find_open_proposal_non_list_payload_fails_open(monkeypatch) -> None: + _install_http(monkeypatch, [FakeResponse(200, {"message": "nope"})]) + assert module.find_open_proposal(_cfg()) == _none_found("open proposal payload is not a list") + + +def test_find_open_proposal_request_exception_fails_open(monkeypatch) -> None: + _install_http(monkeypatch, [RuntimeError("connection reset")]) + assert module.find_open_proposal(_cfg()) == _none_found("open proposal lookup failed: connection reset") + + +def test_find_open_proposal_without_base_url(monkeypatch) -> None: + calls = _install_http(monkeypatch, []) + assert module.find_open_proposal(_cfg(gitea_base_url="")) == _none_found("gitea base url is empty") + assert calls["requests"] == [] + + def test_push_branch_success_with_new_branch_payload(monkeypatch) -> None: calls, result = _push(monkeypatch, [FakeResponse(200, {"sha": "abc123"}), FakeResponse(201, {})]) assert result == {"branch": BRANCH, "committed": True, "error": None} diff --git a/tests/test_hermes_console_evidence.py b/tests/test_hermes_console_evidence.py new file mode 100644 index 0000000..aa48d29 --- /dev/null +++ b/tests/test_hermes_console_evidence.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import pytest + +from ariadne.services import hermes_console_evidence as module + + +def _console(lines) -> str: # type: ignore[no-untyped-def] + return "\n".join(lines) + + +def _noise(count: int, prefix: str = "step") -> list[str]: + return [f"[Pipeline] {prefix} {index} completed ok" for index in range(count)] + + +@pytest.mark.parametrize( + ("line", "marker"), + [ + ("=== FAILURES ===", "=== FAILURES ==="), + ("=========== short test summary info ==========", "short test summary"), + ("ERROR at setup of test_thing", "ERROR at setup"), + ("Traceback (most recent call last):", "Traceback (most recent call last)"), + (" raise AssertionError(x)", "AssertionError"), + ("FAILED tests/test_a.py::test_b - boom", "FAILED "), + ("E assert 1 == 2", "E "), + ("BUILD FAILED in 3s", "BUILD FAILED"), + ("FAILURE: gradle task :check reported problems", "FAILURE:"), + ("process exited with non-zero exit status", "non-zero exit"), + ("script returned exit code 1", "exit code"), + ("bash: pytest: command not found", "command not found"), + ("cp: No such file or directory", "No such file"), + ("java.io.IOException: Exception while archiving", "Exception"), + ("ERROR: something broke", "ERROR:"), + ("error: lowercase also matches", "ERROR:"), + ("Quality gate failed for project", "Quality gate"), + ("Required test coverage of 80% not reached", "coverage"), + ("FAIL Required test --fail-under=80", "fail-under"), + ("missing docstring on public function", "docstring"), + ("Semgrep found 3 findings", "Semgrep"), + ("SonarQube analysis failed", "SonarQube"), + ("Trivy detected HIGH severity", "Trivy"), + ("ruff check found 2 problems", "ruff"), + ("[loc] file exceeds 500 lines", "[loc]"), + ("pod status ImagePullBackOff", "ImagePullBackOff"), + ("container was OOMKilled", "OOMKilled"), + ( + "Failed to establish a new connection: [Errno 111]", + "Failed to establish a new connection", + ), + ( + "Temporary failure in name resolution for jenkins", + "Temporary failure in name resolution", + ), + ], +) +def test_marker_families_are_detected(line: str, marker: str) -> None: + assert module.marker_for_line(line) == marker + result = module.extract_console_evidence(line) + assert result["regions"][0]["marker"] == marker + assert result["regions"][0]["line_number"] == 1 + + +def test_clean_lines_produce_no_regions() -> None: + result = module.extract_console_evidence(_console(_noise(20))) + assert result["regions"] == [] + assert result["truncated"] is False + assert result["total_lines"] == 20 + + +def test_prefix_only_marker_does_not_match_mid_line() -> None: + assert module.marker_for_line("phase done nothing wrong here") is None + assert module.marker_for_line(" E assert False") == "E " + + +def test_prefix_only_failed_marker_ignores_prose() -> None: + assert module.marker_for_line("Quality gate FAILED for project") == "Quality gate" + assert module.marker_for_line(" FAILED tests/test_a.py::test_b") == "FAILED " + + +def test_priority_order_prefers_pytest_signal() -> None: + assert module.marker_for_line("ERROR: raised AssertionError while loading") == "AssertionError" + + +def test_context_window_before_and_after() -> None: + lines = [f"line-{index}" for index in range(60)] + lines[30] = "ERROR: boom" + result = module.extract_console_evidence(_console(lines)) + region = result["regions"][0] + assert region["line_number"] == 31 + text = region["text"].splitlines() + assert text[0] == "line-24" + assert text[-1] == "line-42" + assert len(text) == 19 + + +def test_context_window_is_configurable_and_clamped_at_edges() -> None: + lines = ["ERROR: boom"] + [f"line-{index}" for index in range(3)] + result = module.extract_console_evidence( + _console(lines), {"context_before": 2, "context_after": 1} + ) + assert result["regions"][0]["text"] == "ERROR: boom\nline-0" + + +def test_overlapping_regions_are_merged() -> None: + lines = [f"line-{index}" for index in range(40)] + lines[10] = "ERROR: first" + lines[15] = "ERROR: second" + result = module.extract_console_evidence(_console(lines)) + assert len(result["regions"]) == 1 + region = result["regions"][0] + assert region["line_number"] == 11 + assert "ERROR: first" in region["text"] + assert "ERROR: second" in region["text"] + assert region["text"].splitlines()[-1] == "line-27" + + +def test_distant_regions_are_kept_separate() -> None: + lines = [f"line-{index}" for index in range(200)] + lines[10] = "ERROR: first" + lines[150] = "ERROR: second" + result = module.extract_console_evidence(_console(lines)) + assert [region["line_number"] for region in result["regions"]] == [11, 151] + + +def test_repeated_regions_are_deduplicated_with_a_note() -> None: + block = ["connecting to registry", "ERROR: connection reset by peer (attempt 1)", "retrying"] + lines: list[str] = [] + for attempt in range(5): + lines.extend(_noise(30, "pad")) + lines.extend([block[0], block[1].replace("1", str(attempt + 1)), block[2]]) + lines.extend(_noise(30, "pad")) + result = module.extract_console_evidence(_console(lines)) + assert len(result["regions"]) == 1 + assert result["regions"][0]["text"].endswith("(repeated 5 times)") + assert result["regions"][0]["line_number"] == 32 + + +def test_distinct_regions_are_not_deduplicated() -> None: + lines = _noise(30) + ["ERROR: alpha broke"] + _noise(30, "mid") + ["ERROR: beta broke"] + result = module.extract_console_evidence(_console(lines)) + assert len(result["regions"]) == 2 + assert "repeated" not in result["regions"][1]["text"] + + +def test_byte_budget_prefers_earliest_regions() -> None: + lines: list[str] = [] + for index in range(6): + lines.extend(_noise(40, "pad")) + lines.append(f"ERROR: distinct failure {chr(ord('a') + index)} " + "x" * 400) + result = module.extract_console_evidence(_console(lines), {"max_total_bytes": 2000}) + assert result["truncated"] is True + markers = [region["line_number"] for region in result["regions"]] + assert markers == sorted(markers) + assert result["regions"][0]["line_number"] == 41 + assert "distinct failure a" in result["regions"][0]["text"] + assert len(result["regions"]) < 6 + assert sum(len(region["text"]) for region in result["regions"]) <= 2000 + + +def test_first_region_is_kept_and_clipped_when_it_alone_exceeds_budget() -> None: + lines = ["ERROR: huge " + "y" * 5000, "ERROR: later " + "z" * 5000] + result = module.extract_console_evidence(_console(lines), {"max_total_bytes": 100}) + assert len(result["regions"]) == 1 + assert len(result["regions"][0]["text"]) == 100 + assert result["truncated"] is True + + +def test_zero_byte_budget_still_reports_one_empty_region() -> None: + result = module.extract_console_evidence("ERROR: boom", {"max_total_bytes": 0}) + assert result["regions"] == [{"marker": "ERROR:", "line_number": 1, "text": ""}] + assert result["truncated"] is True + + +def test_max_regions_caps_the_region_count() -> None: + lines: list[str] = [] + for index in range(8): + lines.extend(_noise(40, "pad")) + lines.append(f"ERROR: unique failure {chr(ord('a') + index)}") + result = module.extract_console_evidence(_console(lines), {"max_regions": 3}) + assert len(result["regions"]) == 3 + assert result["truncated"] is True + assert "ERROR: unique failure a" in result["regions"][0]["text"] + + +def test_all_regions_within_budget_are_not_truncated() -> None: + lines = _noise(30) + ["ERROR: only one"] + _noise(30, "post") + result = module.extract_console_evidence(_console(lines)) + assert len(result["regions"]) == 1 + assert result["truncated"] is False + + +def test_tail_is_capped_by_lines_and_bytes() -> None: + lines = [f"line-{index}" for index in range(200)] + result = module.extract_console_evidence(_console(lines)) + tail = result["tail"].splitlines() + assert len(tail) == 40 + assert tail[0] == "line-160" + assert tail[-1] == "line-199" + + wide = module.extract_console_evidence(_console(["q" * 500] * 50), {"tail_lines": 20}) + assert len(wide["tail"]) == 4000 + assert wide["tail"].endswith("q") + + assert module.extract_console_evidence("a\nb\nc", {"tail_lines": 0})["tail"] == "" + assert module.extract_console_evidence("a\nb\nc", {"max_tail_bytes": 0})["tail"] == "" + + +def test_tail_shorter_than_the_window_is_returned_whole() -> None: + result = module.extract_console_evidence("a\nb\nc") + assert result["tail"] == "a\nb\nc" + assert result["total_lines"] == 3 + + +def test_empty_and_none_input() -> None: + for value in ("", None, 12345, b"bytes"): + result = module.extract_console_evidence(value) # type: ignore[arg-type] + assert result == {"regions": [], "tail": "", "total_lines": 0, "truncated": False} + + +def test_invalid_cfg_values_fall_back_to_defaults() -> None: + lines = [f"line-{index}" for index in range(200)] + cfg = {"tail_lines": "not-an-int", "context_before": -5, "max_regions": None} + result = module.extract_console_evidence(_console(lines), cfg) + assert len(result["tail"].splitlines()) == 40 + + numeric = module.extract_console_evidence(_console(lines), {"tail_lines": "7"}) + assert len(numeric["tail"].splitlines()) == 7 + + assert module.extract_console_evidence(_console(lines), "not-a-dict")["total_lines"] == 200 # type: ignore[arg-type] + + +def test_extractor_never_raises(monkeypatch) -> None: # type: ignore[no-untyped-def] + def boom(lines, options): # type: ignore[no-untyped-def] + raise RuntimeError("kaboom") + + monkeypatch.setattr(module, "_regions", boom) + assert module.extract_console_evidence("ERROR: boom") == { + "regions": [], + "tail": "", + "total_lines": 0, + "truncated": False, + } + + +def _long_pipeline_console() -> str: + lines: list[str] = ["Started by user jenkins", "[Pipeline] node", "Running on agent-1"] + lines.extend(_noise(120, "checkout")) + lines.extend( + [ + "[Pipeline] stage: Unit Tests", + "+ python -m pytest tests -q", + "tests/test_wallet.py .........", + "tests/test_ledger.py ..F", + "=================================== FAILURES ===================================", + "____________________ test_ledger_balance_after_migration _______________________", + " def test_ledger_balance_after_migration():", + " balance = ledger.balance('acct-7')", + "> assert balance == 100", + "E assert 0 == 100", + "tests/test_ledger.py:42: AssertionError", + "FAILED tests/test_ledger.py::test_ledger_balance_after_migration", + ] + ) + lines.extend(_noise(3000, "teardown")) + lines.extend( + [ + "[Pipeline] stage: Archive", + "java.io.IOException: Failed to archive artifacts", + " at hudson.FilePath.copyRecursiveTo(FilePath.java:2810)", + "[Pipeline] End of Pipeline", + "ERROR: script returned exit code 1", + "Finished: FAILURE", + ] + ) + return _console(lines) + + +def test_long_pipeline_keeps_the_early_failure_that_the_tail_loses() -> None: + console = _long_pipeline_console() + result = module.extract_console_evidence(console) + + assert result["total_lines"] > 3000 + assert "FAILED tests/test_ledger.py::test_ledger_balance_after_migration" not in result["tail"] + assert "script returned exit code 1" in result["tail"] + + first = result["regions"][0] + assert first["marker"] == "=== FAILURES ===" + assert "test_ledger_balance_after_migration" in first["text"] + assert "assert 0 == 100" in first["text"] + assert "FAILED tests/test_ledger.py::test_ledger_balance_after_migration" in first["text"] + assert first["line_number"] < 200 + assert sum(len(region["text"]) for region in result["regions"]) <= 12000 + + +def test_long_pipeline_regions_stay_chronological_and_reach_the_end() -> None: + result = module.extract_console_evidence(_long_pipeline_console()) + numbers = [region["line_number"] for region in result["regions"]] + assert numbers == sorted(numbers) + assert len(numbers) >= 2 + assert "script returned exit code 1" in result["regions"][-1]["text"]