03f5fbf599
All checks were successful
Tests / Declarative: Post Actions passed: 1438
The sweep opens a pull request only when everything lines up: mapped repository, file inside the write allowlist, and a change expressible as one anchored snippet. Most of this instance's backlog fails the last of those - the bulk of it is cognitive-complexity refactors - so those findings produced nothing at all. That is backwards. A finding nobody can patch automatically is precisely the one a maintainer has to do by hand, which is when knowing the intended fix is worth the most. Such findings now become an issue carrying the finding, links to both the SonarQube entry and the Hermes run, and the code Hermes believes would resolve it. A declined pull request falls through to the same path rather than ending the attempt. Not a patch, and the issue says so: nothing here is anchored, validated against the file, or pushed. That is what lets a suggestion describe work too large or too diffuse for the patcher, which is the whole point of the path. Deduped on the rule through the existing issue marker, so one root cause yields one issue however many files it spans. Off by default: it writes to real repositories, so an operator turns it on deliberately. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
478 lines
16 KiB
Python
478 lines
16 KiB
Python
"""Tests for the scheduled SonarQube quality sweep."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from ariadne.services import hermes_sonar_sweep as module
|
|
|
|
|
|
REPO_CFG = {
|
|
"owner": "bstein",
|
|
"repo": "ariadne",
|
|
"allowed_path_prefixes": ["ariadne/"],
|
|
"allowed_suffixes": [".py"],
|
|
}
|
|
|
|
|
|
def _issue(**overrides):
|
|
issue = {
|
|
"key": "AZ-1",
|
|
"path": "ariadne/services/hermes_code_defects.py",
|
|
"line": 160,
|
|
"rule": "python:S1172",
|
|
"severity": "MAJOR",
|
|
"type": "CODE_SMELL",
|
|
"effort": "5min",
|
|
"message": "Remove the unused function parameter.",
|
|
}
|
|
issue.update(overrides)
|
|
return issue
|
|
|
|
|
|
def _config(**overrides):
|
|
values = {
|
|
"hermes_sonar_projects": {"ariadne": "ariadne"},
|
|
"hermes_sonar_url": "http://sonar:9000",
|
|
"hermes_sonar_token": "t",
|
|
"hermes_sonar_types": ["CODE_SMELL"],
|
|
"hermes_sonar_severities": [],
|
|
"hermes_sonar_max_per_sweep": 1,
|
|
"hermes_sonar_max_effort_minutes": 20,
|
|
"hermes_sonar_timeout_seconds": 5,
|
|
"hermes_sonar_advice_enabled": False,
|
|
}
|
|
values.update(overrides)
|
|
return SimpleNamespace(**values)
|
|
|
|
|
|
class _Storage:
|
|
def __init__(self):
|
|
self.events = []
|
|
|
|
def record_event(self, event_type, detail):
|
|
self.events.append((event_type, detail))
|
|
|
|
|
|
@pytest.fixture
|
|
def wiring(monkeypatch):
|
|
"""Stub the repo resolution, the client and the proposal flow."""
|
|
|
|
calls = {"proposals": [], "issues": [_issue()], "error": None, "result": {"status": "pr_opened"}}
|
|
monkeypatch.setattr(module.hermes_code_flow, "code_config", lambda config: {})
|
|
monkeypatch.setattr(module.hermes_code_flow, "resolve_repo_config", lambda job, cfg: REPO_CFG)
|
|
monkeypatch.setattr(
|
|
module.hermes_sonar_client, "fetch_issues",
|
|
lambda cfg, project: (calls["issues"], calls["error"]),
|
|
)
|
|
|
|
def _propose(storage, **kwargs):
|
|
calls["proposals"].append(kwargs)
|
|
return calls["result"]
|
|
|
|
monkeypatch.setattr(module.hermes_code_flow, "propose_code_fix", _propose)
|
|
monkeypatch.setattr(
|
|
module.hermes_code_repair, "open_proposal_incidents",
|
|
lambda cfg: (calls.get("open_incidents", []), None),
|
|
)
|
|
return calls
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("effort", "expected"),
|
|
[
|
|
("5min", 5),
|
|
("1h", 60),
|
|
("1h30min", 90),
|
|
("2d", 960),
|
|
("", None),
|
|
("soon", None),
|
|
("30", None),
|
|
(None, None),
|
|
],
|
|
)
|
|
def test_the_effort_estimate_is_parsed_into_minutes(effort, expected) -> None:
|
|
assert module.effort_minutes(effort) == expected
|
|
|
|
|
|
def test_the_cheapest_finding_is_chosen_first() -> None:
|
|
"""Effort is the closest proxy for a change the validator can check."""
|
|
|
|
issues = [
|
|
_issue(key="a", effort="15min"),
|
|
_issue(key="b", effort="5min"),
|
|
_issue(key="c", effort="10min"),
|
|
]
|
|
|
|
assert module.select_issue(issues, REPO_CFG, _config())["key"] == "b"
|
|
|
|
|
|
def test_severity_breaks_a_tie_on_effort() -> None:
|
|
issues = [_issue(key="a", severity="MINOR"), _issue(key="b", severity="CRITICAL")]
|
|
|
|
assert module.select_issue(issues, REPO_CFG, _config())["key"] == "b"
|
|
|
|
|
|
def test_the_key_breaks_a_full_tie_so_the_pick_is_stable() -> None:
|
|
"""A sweep that proposes nothing today must retry the same finding."""
|
|
|
|
issues = [_issue(key="zz"), _issue(key="aa")]
|
|
|
|
assert module.select_issue(issues, REPO_CFG, _config())["key"] == "aa"
|
|
assert module.select_issue(list(reversed(issues)), REPO_CFG, _config())["key"] == "aa"
|
|
|
|
|
|
def test_an_expensive_finding_is_left_to_a_person() -> None:
|
|
issues = [_issue(effort="4h")]
|
|
|
|
assert module.select_issue(issues, REPO_CFG, _config()) is None
|
|
|
|
|
|
def test_an_unparseable_effort_is_not_treated_as_free() -> None:
|
|
"""An unreadable estimate is not evidence that the work is small."""
|
|
|
|
assert module.select_issue([_issue(effort="dunno")], REPO_CFG, _config()) is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"path",
|
|
["docs/readme.md", "ariadne/thing.txt", "scripts/tool.py", ""],
|
|
)
|
|
def test_a_finding_outside_the_write_allowlist_is_never_chosen(path) -> None:
|
|
assert module.select_issue([_issue(path=path)], REPO_CFG, _config()) is None
|
|
|
|
|
|
def test_a_sweep_opens_one_proposal_for_the_chosen_finding(wiring) -> None:
|
|
storage = _Storage()
|
|
|
|
result = module.sweep(storage, _config(), {})
|
|
|
|
assert result == {"proposed": 1, "advised": 0, "skipped": [], "projects": 1}
|
|
proposal = wiring["proposals"][0]
|
|
assert proposal["incident_id"] == "sonar/ariadne/python:S1172/AZ-1"
|
|
assert proposal["job"] == "ariadne"
|
|
assert proposal["build_number"] == "sonar-AZ-1"
|
|
|
|
|
|
def test_the_bundle_offers_only_the_file_the_finding_names() -> None:
|
|
"""The finding states the file; ranking console text alongside adds noise."""
|
|
|
|
bundle = module.bundle_for("ariadne", _issue())
|
|
|
|
assert bundle["sonarqube"] == {"project": "ariadne", "issues": [_issue()]}
|
|
assert bundle["jenkins"]["console_failures"] == []
|
|
assert bundle["jenkins"]["console_tail"] == ""
|
|
assert bundle["incident_id"] == "sonar/ariadne/python:S1172/AZ-1"
|
|
|
|
|
|
def test_the_sweep_budget_stops_after_its_quota(wiring) -> None:
|
|
"""A backlog must not become a pull request queue nobody drains."""
|
|
|
|
config = _config(hermes_sonar_projects={"a": "a", "b": "b", "c": "c"})
|
|
|
|
result = module.sweep(_Storage(), config, {})
|
|
|
|
assert result["proposed"] == 1
|
|
assert len(wiring["proposals"]) == 1
|
|
assert any("budget spent" in item for item in result["skipped"])
|
|
|
|
|
|
def test_a_declined_proposal_is_reported_and_does_not_spend_budget(wiring) -> None:
|
|
wiring["result"] = {"status": "human_required", "reason": "open_proposal_limit_reached"}
|
|
config = _config(hermes_sonar_projects={"a": "a", "b": "b"})
|
|
|
|
result = module.sweep(_Storage(), config, {})
|
|
|
|
assert result["proposed"] == 0
|
|
assert len(wiring["proposals"]) == 2
|
|
assert any("open_proposal_limit_reached" in item for item in result["skipped"])
|
|
|
|
|
|
def test_no_configured_projects_makes_no_call(wiring) -> None:
|
|
result = module.sweep(_Storage(), _config(hermes_sonar_projects={}), {})
|
|
|
|
assert result == {"proposed": 0, "advised": 0, "skipped": ["no_projects_configured"], "projects": 0}
|
|
assert wiring["proposals"] == []
|
|
|
|
|
|
def test_a_project_naming_no_repository_is_skipped(monkeypatch, wiring) -> None:
|
|
monkeypatch.setattr(module.hermes_code_flow, "resolve_repo_config", lambda job, cfg: None)
|
|
|
|
result = module.sweep(_Storage(), _config(), {})
|
|
|
|
assert result["proposed"] == 0
|
|
assert "maps to no repository" in result["skipped"][0]
|
|
assert wiring["proposals"] == []
|
|
|
|
|
|
def test_a_fetch_error_is_reported_not_raised(wiring) -> None:
|
|
wiring["error"] = "sonar fetch http 503"
|
|
|
|
result = module.sweep(_Storage(), _config(), {})
|
|
|
|
assert result["proposed"] == 0
|
|
assert result["skipped"] == ["ariadne: sonar fetch http 503"]
|
|
|
|
|
|
def test_a_finding_too_large_to_patch_becomes_advice(wiring) -> None:
|
|
"""The bulk of the backlog is refactors no anchored patch can express."""
|
|
|
|
wiring["issues"] = [_issue(effort="8h")]
|
|
|
|
result = module.sweep(_Storage(), _config(), {})
|
|
|
|
assert result["proposed"] == 0
|
|
assert result["skipped"] == ["ariadne: too large to patch (advice disabled)"]
|
|
assert wiring["proposals"] == []
|
|
|
|
|
|
def test_a_sweep_never_takes_down_the_scheduler(monkeypatch) -> None:
|
|
"""Triage shares this scheduler; a background sweep must not stop it."""
|
|
|
|
monkeypatch.setattr(
|
|
module.hermes_code_flow, "code_config",
|
|
lambda config: (_ for _ in ()).throw(RuntimeError("boom")),
|
|
)
|
|
|
|
result = module.sweep(_Storage(), _config(), {})
|
|
|
|
assert result["proposed"] == 0
|
|
assert "sweep_failed" in result["skipped"][0]
|
|
|
|
|
|
def test_a_project_with_no_job_maps_to_itself() -> None:
|
|
assert module._projects(_config(hermes_sonar_projects={"ariadne": ""})) == {
|
|
"ariadne": "ariadne"
|
|
}
|
|
assert module._projects(SimpleNamespace(hermes_sonar_projects="nope")) == {}
|
|
|
|
|
|
def test_the_client_config_is_built_from_settings() -> None:
|
|
cfg = module.client_config(_config())
|
|
|
|
assert cfg["sonar_base_url"] == "http://sonar:9000"
|
|
assert cfg["sonar_token"] == "t"
|
|
assert cfg["sonar_types"] == ["CODE_SMELL"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("attr", "func", "bad", "default"),
|
|
[
|
|
("hermes_sonar_max_per_sweep", "_max_per_sweep", "junk", module.DEFAULT_MAX_PER_SWEEP),
|
|
(
|
|
"hermes_sonar_max_effort_minutes",
|
|
"_max_effort_minutes",
|
|
"junk",
|
|
module.DEFAULT_MAX_EFFORT_MINUTES,
|
|
),
|
|
],
|
|
)
|
|
def test_an_unparseable_bound_falls_back_to_its_default(attr, func, bad, default) -> None:
|
|
assert getattr(module, func)(_config(**{attr: bad})) == default
|
|
|
|
|
|
def test_the_bounds_are_floored() -> None:
|
|
assert module._max_per_sweep(_config(hermes_sonar_max_per_sweep=-4)) == 0
|
|
assert module._max_effort_minutes(_config(hermes_sonar_max_effort_minutes=0)) == 1
|
|
|
|
|
|
def test_the_sweep_is_disabled_until_an_operator_turns_it_on(monkeypatch) -> None:
|
|
"""It opens pull requests nobody asked for; that is a deliberate choice."""
|
|
|
|
monkeypatch.setattr(module, "settings", SimpleNamespace(hermes_sonar_enabled=False))
|
|
|
|
assert module.run_hermes_sonar_sweep(_Storage()) == {"status": "disabled"}
|
|
|
|
|
|
def test_the_default_deployment_leaves_it_off() -> None:
|
|
"""A fresh deployment must not start opening pull requests on its own."""
|
|
|
|
from ariadne.settings import settings as real_settings
|
|
|
|
assert real_settings.hermes_sonar_enabled is False
|
|
|
|
|
|
def test_the_scheduled_entry_point_runs_the_sweep_when_enabled(monkeypatch) -> None:
|
|
seen = {}
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
hermes_sonar_enabled=True,
|
|
hermes_api_url="http://hermes:8642",
|
|
hermes_api_key="k",
|
|
hermes_run_timeout_seconds=420.0,
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
module, "sweep", lambda storage, config, cfg: seen.update(cfg=cfg) or {"proposed": 0}
|
|
)
|
|
|
|
assert module.run_hermes_sonar_sweep(_Storage()) == {"proposed": 0}
|
|
assert seen["cfg"] == {
|
|
"base_url": "http://hermes:8642",
|
|
"api_key": "k",
|
|
"total_timeout_seconds": 420.0,
|
|
}
|
|
|
|
|
|
def test_the_sweep_prompt_never_claims_a_build_failed() -> None:
|
|
"""A green build framed as a failing test is wrong in the first line read."""
|
|
|
|
from ariadne.services import hermes_code_prompt
|
|
|
|
cfg = {"owner": "bstein", "repo": "ariadne", "base_branch": "master"}
|
|
prompt = hermes_code_prompt.build_patch_prompt(
|
|
"sonar/ariadne/AZ-1",
|
|
module.bundle_for("ariadne", _issue()),
|
|
{**cfg, "allowed_path_prefixes": ["ariadne/"], "allowed_suffixes": [".py"]},
|
|
{"ariadne/services/hermes_code_defects.py": "x"},
|
|
)
|
|
|
|
assert prompt.startswith("A static-analysis finding, not a build failure.")
|
|
assert "Failing test evidence bundle" not in prompt
|
|
assert "Static analysis evidence bundle:" in prompt
|
|
assert "The build reported these specific defects" not in prompt
|
|
# The triage skill investigates a failed build; there isn't one.
|
|
assert "$triage-titan-test-failures" not in prompt
|
|
|
|
|
|
def test_a_build_driven_prompt_is_unchanged() -> None:
|
|
from ariadne.services import hermes_code_prompt
|
|
|
|
bundle = {"jenkins": {"console_tail": "AssertionError", "console_failures": [], "failed_tests": []}}
|
|
prompt = hermes_code_prompt.build_patch_prompt(
|
|
"ariadne/9", bundle, {"owner": "bstein", "repo": "ariadne"}, {"ariadne/a.py": "x"}
|
|
)
|
|
|
|
assert prompt.startswith("Use $triage-titan-test-failures.")
|
|
assert "Failing test evidence bundle:" in prompt
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("bundle", "expected"),
|
|
[
|
|
({"sonarqube": {"issues": [{"path": "a.py"}]}, "jenkins": {}}, True),
|
|
({"sonarqube": {"issues": []}, "jenkins": {}}, False),
|
|
({"jenkins": {"console_tail": "boom"}}, False),
|
|
# Both present: a real failure is the more urgent framing.
|
|
(
|
|
{"sonarqube": {"issues": [{"path": "a.py"}]}, "jenkins": {"console_tail": "boom"}},
|
|
False,
|
|
),
|
|
("not-a-dict", False),
|
|
],
|
|
)
|
|
def test_a_bundles_origin_decides_the_framing(bundle, expected) -> None:
|
|
from ariadne.services import hermes_code_prompt
|
|
|
|
assert hermes_code_prompt.is_quality_sweep(bundle) is expected
|
|
|
|
|
|
def test_a_rule_already_under_review_is_not_proposed_again(wiring) -> None:
|
|
"""One rule is one root cause; thirty near-identical PRs get read as none."""
|
|
|
|
wiring["open_incidents"] = ["sonar/ariadne/python:S1172/AZ-OTHER"]
|
|
wiring["issues"] = [
|
|
_issue(key="a", rule="python:S1172"),
|
|
_issue(key="b", rule="python:S2208", effort="10min"),
|
|
]
|
|
|
|
result = module.sweep(_Storage(), _config(), {})
|
|
|
|
assert result["proposed"] == 1
|
|
assert wiring["proposals"][0]["incident_id"] == "sonar/ariadne/python:S2208/b"
|
|
|
|
|
|
def test_every_rule_under_review_means_nothing_new_to_propose(wiring) -> None:
|
|
wiring["open_incidents"] = ["sonar/ariadne/python:S1172/AZ-OTHER"]
|
|
wiring["issues"] = [_issue(rule="python:S1172")]
|
|
|
|
result = module.sweep(_Storage(), _config(), {})
|
|
|
|
assert result["proposed"] == 0
|
|
assert result["skipped"] == ["ariadne: no new finding to act on"]
|
|
|
|
|
|
def test_another_projects_open_proposal_does_not_block_this_one(wiring) -> None:
|
|
wiring["open_incidents"] = ["sonar/metis/python:S1172/AZ-X", "ariadne/408"]
|
|
|
|
assert module.sweep(_Storage(), _config(), {})["proposed"] == 1
|
|
|
|
|
|
def test_the_rules_under_review_are_read_from_the_open_proposals(monkeypatch) -> None:
|
|
monkeypatch.setattr(
|
|
module.hermes_code_repair, "open_proposal_incidents",
|
|
lambda cfg: (
|
|
[
|
|
"sonar/ariadne/python:S2208/AZ1",
|
|
"sonar/ariadne/python:S3776/AZ2",
|
|
"sonar/other/python:S1172/AZ3",
|
|
"ariadne/408",
|
|
"sonar/ariadne/malformed",
|
|
],
|
|
None,
|
|
),
|
|
)
|
|
|
|
assert module.proposed_rules(REPO_CFG, "ariadne") == {"python:S2208", "python:S3776"}
|
|
|
|
|
|
def test_an_unreadable_proposal_list_costs_a_duplicate_not_a_silent_rule(monkeypatch) -> None:
|
|
"""Failing open here means one extra PR; failing closed loses a whole rule."""
|
|
|
|
monkeypatch.setattr(
|
|
module.hermes_code_repair, "open_proposal_incidents", lambda cfg: ([], "http 500")
|
|
)
|
|
|
|
assert module.proposed_rules(REPO_CFG, "ariadne") == set()
|
|
|
|
|
|
def test_advice_files_an_issue_when_no_patch_was_possible(monkeypatch, wiring) -> None:
|
|
"""A finding nobody can patch is exactly the one worth explaining."""
|
|
|
|
filed = []
|
|
monkeypatch.setattr(
|
|
module.hermes_sonar_advice, "advise",
|
|
lambda storage, config, hermes_cfg, repo_cfg, project, issue: filed.append(issue)
|
|
or {"filed": True, "reason": "issue_filed", "url": "https://scm/issues/9"},
|
|
)
|
|
wiring["issues"] = [_issue(effort="8h", key="big")]
|
|
|
|
result = module.sweep(_Storage(), _config(hermes_sonar_advice_enabled=True), {})
|
|
|
|
assert result["advised"] == 1
|
|
assert filed[0]["key"] == "big"
|
|
|
|
|
|
def test_a_declined_pull_request_falls_through_to_advice(monkeypatch, wiring) -> None:
|
|
"""A declined proposal is not a dead end; the finding is still real."""
|
|
|
|
monkeypatch.setattr(
|
|
module.hermes_sonar_advice, "advise",
|
|
lambda *a, **k: {"filed": True, "reason": "issue_filed", "url": "u"},
|
|
)
|
|
wiring["result"] = {"status": "human_required", "reason": "patch_rejected: original_ambiguous"}
|
|
|
|
result = module.sweep(_Storage(), _config(hermes_sonar_advice_enabled=True), {})
|
|
|
|
assert result["advised"] == 1
|
|
|
|
|
|
def test_advice_respects_the_rules_already_under_review(wiring) -> None:
|
|
wiring["open_incidents"] = ["sonar/ariadne/python:S1172/OTHER"]
|
|
wiring["issues"] = [_issue(effort="8h", rule="python:S1172")]
|
|
|
|
result = module.sweep(_Storage(), _config(hermes_sonar_advice_enabled=True), {})
|
|
|
|
assert result["skipped"] == ["ariadne: no new finding to act on"]
|
|
|
|
|
|
def test_advice_stays_inside_the_write_allowlist() -> None:
|
|
"""Advising on a file nobody would edit is noise, not help."""
|
|
|
|
assert module.select_advice_issue([_issue(path="docs/x.md", effort="8h")], REPO_CFG, _config()) is None
|
|
assert module.select_advice_issue([_issue(effort="8h")], REPO_CFG, _config())["key"] == "AZ-1"
|