feat(hermes): search a service's own namespace for log evidence
All checks were successful
Tests / Declarative: Post Actions passed: 1200
All checks were successful
Tests / Declarative: Post Actions passed: 1200
Log evidence covered the demo namespace plus jenkins. That is right for the common case, since CI failures happen in Jenkins agent pods, but it means a build failure that correlates with the service itself being unhealthy carries no trace of the service at all. ARIADNE_HERMES_JOB_NAMESPACES maps a job to its own namespace, which is added alongside jenkins rather than replacing it. Unmapped jobs are unchanged, and a namespace already in the list is never duplicated. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
c9c205484e
commit
136caa8477
@ -20,6 +20,8 @@ _MAX_FAILED_TESTS = 10
|
|||||||
_MAX_ERROR_DETAILS_CHARS = 2000
|
_MAX_ERROR_DETAILS_CHARS = 2000
|
||||||
_FAILED_TEST_STATUSES = {"FAILED", "REGRESSION"}
|
_FAILED_TEST_STATUSES = {"FAILED", "REGRESSION"}
|
||||||
_TEST_REPORT_TREE = "suites[cases[className,name,status,errorDetails]]"
|
_TEST_REPORT_TREE = "suites[cases[className,name,status,errorDetails]]"
|
||||||
|
# Jenkins agent pods are where CI failures actually happen, so this is the
|
||||||
|
# namespace that matters for every job and is always included.
|
||||||
_LOG_EXTRA_NAMESPACES = ("jenkins",)
|
_LOG_EXTRA_NAMESPACES = ("jenkins",)
|
||||||
|
|
||||||
|
|
||||||
@ -62,7 +64,7 @@ def collect_evidence(incident_id: str, job: str, last_build: dict) -> dict[str,
|
|||||||
"incident_id": incident_id,
|
"incident_id": incident_id,
|
||||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
"jenkins": jenkins,
|
"jenkins": jenkins,
|
||||||
"log_evidence": collect_log_evidence(_log_config(), incident_id, window_start, window_end),
|
"log_evidence": collect_log_evidence(_log_config(job), incident_id, window_start, window_end),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -218,13 +220,24 @@ def _client_kwargs() -> dict[str, Any]:
|
|||||||
return kwargs
|
return kwargs
|
||||||
|
|
||||||
|
|
||||||
def _log_config() -> dict[str, Any]:
|
def _log_config(job: str = "") -> dict[str, Any]:
|
||||||
"""Build the config passed to the frozen OpenSearch log collector."""
|
"""Build the config passed to the frozen OpenSearch log collector.
|
||||||
|
|
||||||
|
Inputs: the job under triage, used to add that service's own namespace.
|
||||||
|
Outputs: the collector cfg. A build failure sometimes correlates with the
|
||||||
|
service itself being unhealthy, and its runtime logs are the only place
|
||||||
|
that shows; without a mapping only the demo and Jenkins namespaces are
|
||||||
|
searched, which says nothing about the service.
|
||||||
|
"""
|
||||||
|
|
||||||
|
extra = list(_LOG_EXTRA_NAMESPACES)
|
||||||
|
own = (getattr(settings, "hermes_job_namespaces", None) or {}).get(job)
|
||||||
|
if own and own not in extra:
|
||||||
|
extra.append(own)
|
||||||
return {
|
return {
|
||||||
"opensearch_url": settings.opensearch_url,
|
"opensearch_url": settings.opensearch_url,
|
||||||
"namespace": settings.hermes_demo_namespace,
|
"namespace": settings.hermes_demo_namespace,
|
||||||
"extra_namespaces": list(_LOG_EXTRA_NAMESPACES),
|
"extra_namespaces": extra,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -191,6 +191,7 @@ class Settings:
|
|||||||
hermes_min_confidence: float
|
hermes_min_confidence: float
|
||||||
hermes_hung_build_minutes: float
|
hermes_hung_build_minutes: float
|
||||||
hermes_max_branches: int
|
hermes_max_branches: int
|
||||||
|
hermes_job_namespaces: dict
|
||||||
hermes_max_actions_per_incident: int
|
hermes_max_actions_per_incident: int
|
||||||
hermes_api_url: str
|
hermes_api_url: str
|
||||||
hermes_api_key: str
|
hermes_api_key: str
|
||||||
|
|||||||
@ -291,6 +291,7 @@ def _hermes_autotriage_config() -> dict[str, Any]:
|
|||||||
"hermes_min_confidence": _env_float("ARIADNE_HERMES_MIN_CONFIDENCE", 0.85),
|
"hermes_min_confidence": _env_float("ARIADNE_HERMES_MIN_CONFIDENCE", 0.85),
|
||||||
"hermes_hung_build_minutes": _env_float("ARIADNE_HERMES_HUNG_BUILD_MINUTES", 45.0),
|
"hermes_hung_build_minutes": _env_float("ARIADNE_HERMES_HUNG_BUILD_MINUTES", 45.0),
|
||||||
"hermes_max_branches": _env_int("ARIADNE_HERMES_MAX_BRANCHES", 5),
|
"hermes_max_branches": _env_int("ARIADNE_HERMES_MAX_BRANCHES", 5),
|
||||||
|
"hermes_job_namespaces": _pair_map(_env("ARIADNE_HERMES_JOB_NAMESPACES", "")),
|
||||||
"hermes_max_actions_per_incident": _env_int("ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT", 1),
|
"hermes_max_actions_per_incident": _env_int("ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT", 1),
|
||||||
"hermes_api_url": _env(
|
"hermes_api_url": _env(
|
||||||
"ARIADNE_HERMES_API_URL",
|
"ARIADNE_HERMES_API_URL",
|
||||||
|
|||||||
@ -60,6 +60,7 @@ def _settings(**overrides) -> SimpleNamespace: # type: ignore[no-untyped-def]
|
|||||||
"hermes_min_confidence": 0.85,
|
"hermes_min_confidence": 0.85,
|
||||||
"hermes_hung_build_minutes": 45.0,
|
"hermes_hung_build_minutes": 45.0,
|
||||||
"hermes_max_branches": 5,
|
"hermes_max_branches": 5,
|
||||||
|
"hermes_job_namespaces": {},
|
||||||
"hermes_max_actions_per_incident": 1,
|
"hermes_max_actions_per_incident": 1,
|
||||||
"hermes_api_url": "http://hermes:8642",
|
"hermes_api_url": "http://hermes:8642",
|
||||||
"hermes_api_key": "key",
|
"hermes_api_key": "key",
|
||||||
|
|||||||
@ -340,3 +340,35 @@ def test_signature_absent() -> None:
|
|||||||
|
|
||||||
def test_signature_tolerates_malformed_bundle() -> None:
|
def test_signature_tolerates_malformed_bundle() -> None:
|
||||||
assert module.evidence_has_signature({}, INCIDENT_ID) is False
|
assert module.evidence_has_signature({}, INCIDENT_ID) is False
|
||||||
|
|
||||||
|
|
||||||
|
def _settings_with(namespaces): # type: ignore[no-untyped-def]
|
||||||
|
cfg = _settings()
|
||||||
|
cfg.hermes_job_namespaces = namespaces
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_namespaces_always_include_jenkins(monkeypatch) -> None:
|
||||||
|
"""CI failures happen in Jenkins agent pods, whatever the service."""
|
||||||
|
|
||||||
|
monkeypatch.setattr(module, "settings", _settings_with({}))
|
||||||
|
assert module._log_config("lesavka")["extra_namespaces"] == ["jenkins"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_mapped_job_also_searches_its_own_namespace(monkeypatch) -> None:
|
||||||
|
"""A build failure can correlate with the service itself being unhealthy."""
|
||||||
|
|
||||||
|
monkeypatch.setattr(module, "settings", _settings_with({"bstein-dev-home": "bstein-dev-home"}))
|
||||||
|
spaces = module._log_config("bstein-dev-home")["extra_namespaces"]
|
||||||
|
assert "bstein-dev-home" in spaces
|
||||||
|
assert "jenkins" in spaces
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unmapped_job_adds_nothing(monkeypatch) -> None:
|
||||||
|
monkeypatch.setattr(module, "settings", _settings_with({"other": "other-ns"}))
|
||||||
|
assert module._log_config("lesavka")["extra_namespaces"] == ["jenkins"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_namespace_is_never_duplicated(monkeypatch) -> None:
|
||||||
|
monkeypatch.setattr(module, "settings", _settings_with({"j": "jenkins"}))
|
||||||
|
assert module._log_config("j")["extra_namespaces"].count("jenkins") == 1
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user