feat(hermes): search a service's own namespace for log evidence
All checks were successful
Tests / Declarative: Post Actions passed: 1200

Log evidence covered the demo namespace plus jenkins. That is right for the
common case, since CI failures happen in Jenkins agent pods, but it means a
build failure that correlates with the service itself being unhealthy carries
no trace of the service at all.

ARIADNE_HERMES_JOB_NAMESPACES maps a job to its own namespace, which is added
alongside jenkins rather than replacing it. Unmapped jobs are unchanged, and a
namespace already in the list is never duplicated.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
codex 2026-08-06 12:32:59 -03:00
parent c9c205484e
commit 136caa8477
5 changed files with 52 additions and 4 deletions

View File

@ -20,6 +20,8 @@ _MAX_FAILED_TESTS = 10
_MAX_ERROR_DETAILS_CHARS = 2000 _MAX_ERROR_DETAILS_CHARS = 2000
_FAILED_TEST_STATUSES = {"FAILED", "REGRESSION"} _FAILED_TEST_STATUSES = {"FAILED", "REGRESSION"}
_TEST_REPORT_TREE = "suites[cases[className,name,status,errorDetails]]" _TEST_REPORT_TREE = "suites[cases[className,name,status,errorDetails]]"
# Jenkins agent pods are where CI failures actually happen, so this is the
# namespace that matters for every job and is always included.
_LOG_EXTRA_NAMESPACES = ("jenkins",) _LOG_EXTRA_NAMESPACES = ("jenkins",)
@ -62,7 +64,7 @@ def collect_evidence(incident_id: str, job: str, last_build: dict) -> dict[str,
"incident_id": incident_id, "incident_id": incident_id,
"generated_at": datetime.now(timezone.utc).isoformat(), "generated_at": datetime.now(timezone.utc).isoformat(),
"jenkins": jenkins, "jenkins": jenkins,
"log_evidence": collect_log_evidence(_log_config(), incident_id, window_start, window_end), "log_evidence": collect_log_evidence(_log_config(job), incident_id, window_start, window_end),
} }
@ -218,13 +220,24 @@ def _client_kwargs() -> dict[str, Any]:
return kwargs return kwargs
def _log_config() -> dict[str, Any]: def _log_config(job: str = "") -> dict[str, Any]:
"""Build the config passed to the frozen OpenSearch log collector.""" """Build the config passed to the frozen OpenSearch log collector.
Inputs: the job under triage, used to add that service's own namespace.
Outputs: the collector cfg. A build failure sometimes correlates with the
service itself being unhealthy, and its runtime logs are the only place
that shows; without a mapping only the demo and Jenkins namespaces are
searched, which says nothing about the service.
"""
extra = list(_LOG_EXTRA_NAMESPACES)
own = (getattr(settings, "hermes_job_namespaces", None) or {}).get(job)
if own and own not in extra:
extra.append(own)
return { return {
"opensearch_url": settings.opensearch_url, "opensearch_url": settings.opensearch_url,
"namespace": settings.hermes_demo_namespace, "namespace": settings.hermes_demo_namespace,
"extra_namespaces": list(_LOG_EXTRA_NAMESPACES), "extra_namespaces": extra,
} }

View File

@ -191,6 +191,7 @@ class Settings:
hermes_min_confidence: float hermes_min_confidence: float
hermes_hung_build_minutes: float hermes_hung_build_minutes: float
hermes_max_branches: int hermes_max_branches: int
hermes_job_namespaces: dict
hermes_max_actions_per_incident: int hermes_max_actions_per_incident: int
hermes_api_url: str hermes_api_url: str
hermes_api_key: str hermes_api_key: str

View File

@ -291,6 +291,7 @@ def _hermes_autotriage_config() -> dict[str, Any]:
"hermes_min_confidence": _env_float("ARIADNE_HERMES_MIN_CONFIDENCE", 0.85), "hermes_min_confidence": _env_float("ARIADNE_HERMES_MIN_CONFIDENCE", 0.85),
"hermes_hung_build_minutes": _env_float("ARIADNE_HERMES_HUNG_BUILD_MINUTES", 45.0), "hermes_hung_build_minutes": _env_float("ARIADNE_HERMES_HUNG_BUILD_MINUTES", 45.0),
"hermes_max_branches": _env_int("ARIADNE_HERMES_MAX_BRANCHES", 5), "hermes_max_branches": _env_int("ARIADNE_HERMES_MAX_BRANCHES", 5),
"hermes_job_namespaces": _pair_map(_env("ARIADNE_HERMES_JOB_NAMESPACES", "")),
"hermes_max_actions_per_incident": _env_int("ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT", 1), "hermes_max_actions_per_incident": _env_int("ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT", 1),
"hermes_api_url": _env( "hermes_api_url": _env(
"ARIADNE_HERMES_API_URL", "ARIADNE_HERMES_API_URL",

View File

@ -60,6 +60,7 @@ def _settings(**overrides) -> SimpleNamespace: # type: ignore[no-untyped-def]
"hermes_min_confidence": 0.85, "hermes_min_confidence": 0.85,
"hermes_hung_build_minutes": 45.0, "hermes_hung_build_minutes": 45.0,
"hermes_max_branches": 5, "hermes_max_branches": 5,
"hermes_job_namespaces": {},
"hermes_max_actions_per_incident": 1, "hermes_max_actions_per_incident": 1,
"hermes_api_url": "http://hermes:8642", "hermes_api_url": "http://hermes:8642",
"hermes_api_key": "key", "hermes_api_key": "key",

View File

@ -340,3 +340,35 @@ def test_signature_absent() -> None:
def test_signature_tolerates_malformed_bundle() -> None: def test_signature_tolerates_malformed_bundle() -> None:
assert module.evidence_has_signature({}, INCIDENT_ID) is False assert module.evidence_has_signature({}, INCIDENT_ID) is False
def _settings_with(namespaces): # type: ignore[no-untyped-def]
cfg = _settings()
cfg.hermes_job_namespaces = namespaces
return cfg
def test_log_namespaces_always_include_jenkins(monkeypatch) -> None:
"""CI failures happen in Jenkins agent pods, whatever the service."""
monkeypatch.setattr(module, "settings", _settings_with({}))
assert module._log_config("lesavka")["extra_namespaces"] == ["jenkins"]
def test_a_mapped_job_also_searches_its_own_namespace(monkeypatch) -> None:
"""A build failure can correlate with the service itself being unhealthy."""
monkeypatch.setattr(module, "settings", _settings_with({"bstein-dev-home": "bstein-dev-home"}))
spaces = module._log_config("bstein-dev-home")["extra_namespaces"]
assert "bstein-dev-home" in spaces
assert "jenkins" in spaces
def test_an_unmapped_job_adds_nothing(monkeypatch) -> None:
monkeypatch.setattr(module, "settings", _settings_with({"other": "other-ns"}))
assert module._log_config("lesavka")["extra_namespaces"] == ["jenkins"]
def test_a_namespace_is_never_duplicated(monkeypatch) -> None:
monkeypatch.setattr(module, "settings", _settings_with({"j": "jenkins"}))
assert module._log_config("j")["extra_namespaces"].count("jenkins") == 1