"""Tests for recognising mechanically-fixable defects in console evidence.""" from __future__ import annotations from ariadne.services import hermes_code_defects as module CFG = {"allowed_path_prefixes": ["ariadne/", "src/"], "allowed_suffixes": [".py", ".go", ".ts"]} def _bundle(*lines: str) -> dict: return {"jenkins": {"console_failures": [{"text": "\n".join(lines)}], "console_tail": ""}} def test_ruff_diagnostic_is_extracted_whole() -> None: found = module.extract_lint_defects( _bundle("ariadne/app.py:12:5: E501 Line too long (95 > 88)"), CFG ) assert found == [ { "category": module.LINT_VIOLATION, "tool": "ruff", "path": "ariadne/app.py", "line": 12, "rule": "E501", "message": "Line too long (95 > 88)", } ] def test_golangci_diagnostic_is_extracted() -> None: found = module.extract_lint_defects(_bundle("src/main.go:9:2: unused variable x (govet)"), CFG) assert found[0]["tool"] == "golangci" assert (found[0]["path"], found[0]["line"], found[0]["rule"]) == ("src/main.go", 9, "govet") def test_eslint_diagnostics_inherit_the_preceding_file_header() -> None: """eslint prints the path once, then indented diagnostics beneath it.""" found = module.extract_lint_defects( _bundle("src/panel.ts", " 12:5 error Unexpected console statement no-console"), CFG ) assert (found[0]["path"], found[0]["line"], found[0]["rule"]) == ("src/panel.ts", 12, "no-console") def test_a_file_outside_the_write_allowlist_is_dropped() -> None: """Reporting a defect the patcher could never write is worse than silence.""" assert module.extract_lint_defects(_bundle("vendor/x.py:1:1: E501 Line too long"), CFG) == [] assert module.extract_lint_defects(_bundle("ariadne/x.rb:1:1: E501 Line too long"), CFG) == [] def test_duplicate_diagnostics_are_collapsed() -> None: line = "ariadne/app.py:12:5: E501 Line too long (95 > 88)" assert len(module.extract_lint_defects(_bundle(line, line, line), CFG)) == 1 def test_extraction_is_bounded() -> None: lines = [f"ariadne/f{i}.py:{i}:1: E501 Line too long" for i in range(50)] assert len(module.extract_lint_defects(_bundle(*lines), CFG)) == module._MAX_DEFECTS def test_ordinary_console_noise_matches_nothing() -> None: noise = _bundle("Running tests...", "OK", "+ ruff check ariadne", "All checks passed!") assert module.extract_lint_defects(noise, CFG) == [] def test_never_raises_on_hostile_input() -> None: for bad in (None, {}, {"jenkins": None}, {"jenkins": {"console_failures": "no"}}, 7): assert module.extract_lint_defects(bad, CFG) == [] assert module.extract_lint_defects(_bundle("ariadne/a.py:1:1: E1 x"), None) == [] def test_instruction_names_rule_and_line() -> None: defects = module.extract_lint_defects(_bundle("ariadne/app.py:12:5: E501 Line too long"), CFG) text = module.defect_instruction(defects) assert "ariadne/app.py line 12: E501" in text assert "do not suppress the rule" in text def test_no_defects_yields_no_instruction() -> None: """The caller must fall back to the open-ended request, not an empty header.""" assert module.defect_instruction([]) == "" def test_the_patch_prompt_carries_the_linter_diagnosis(monkeypatch) -> None: """The instruction must reach Hermes, not just be computed.""" from ariadne.services import hermes_code_prompt bundle = { "jenkins": { "console_failures": [{"text": "ariadne/app.py:12:5: E501 Line too long (95 > 88)"}], "console_tail": "", } } cfg = { "owner": "bstein", "repo": "ariadne", "base_branch": "master", "allowed_path_prefixes": ["ariadne/"], "allowed_suffixes": [".py"], } prompt = hermes_code_prompt.build_patch_prompt("ariadne/1", bundle, cfg, {"ariadne/app.py": "x"}) assert "ariadne/app.py line 12: E501" in prompt assert "do not suppress the rule" in prompt def test_the_patch_prompt_is_unchanged_without_diagnostics() -> None: """A build with no linter output must fall back to the open request.""" from ariadne.services import hermes_code_prompt bundle = {"jenkins": {"console_failures": [{"text": "boom"}], "console_tail": ""}} cfg = {"owner": "b", "repo": "r", "base_branch": "main", "allowed_path_prefixes": ["ariadne/"], "allowed_suffixes": [".py"]} prompt = hermes_code_prompt.build_patch_prompt("a/1", bundle, cfg, {"ariadne/a.py": "x"}) assert "A linter reported these violations" not in prompt assert "MINIMAL source fix" in prompt # --- category 2: undefined or misspelled names --------------------------------- def test_python_nameerror_takes_its_file_from_the_traceback_frame() -> None: """The error line names the symbol; the frame above names the file.""" found = module.extract_name_defects( _bundle(' File "ariadne/app.py", line 42, in build', "NameError: name 'setttings' is not defined"), CFG, ) assert found[0]["symbol"] == "setttings" assert (found[0]["path"], found[0]["line"]) == ("ariadne/app.py", 42) assert found[0]["category"] == module.UNDEFINED_NAME def test_import_errors_are_recognised() -> None: for text in ( "ModuleNotFoundError: No module named 'httpx'", "ImportError: cannot import name 'safe_error' from 'ariadne.utils.errors'", ): found = module.extract_name_defects( _bundle(' File "ariadne/app.py", line 3, in ', text), CFG ) assert found, text assert found[0]["category"] == module.UNDEFINED_NAME def test_go_undefined_names_carry_their_own_location() -> None: found = module.extract_name_defects(_bundle("src/main.go:12:5: undefined: doThing"), CFG) assert (found[0]["symbol"], found[0]["path"], found[0]["line"]) == ("doThing", "src/main.go", 12) def test_a_name_defect_without_a_known_file_is_dropped() -> None: """With no frame there is no file, so the patcher could not act on it.""" assert module.extract_name_defects(_bundle("NameError: name 'x' is not defined"), CFG) == [] def test_name_defects_outside_the_write_allowlist_are_dropped() -> None: found = module.extract_name_defects( _bundle(' File "vendor/thing.py", line 1, in x', "NameError: name 'y' is not defined"), CFG ) assert found == [] # --- category 3: failing assertions -------------------------------------------- def test_failing_tests_come_from_structured_results() -> None: bundle = { "jenkins": { "console_failures": [], "console_tail": "", "failed_tests": [ { "name": "test_discount", "className": "tests.test_discount", "errorDetails": "AssertionError: assert 0.0 == 90.0", } ], } } found = module.extract_assertion_defects(bundle, CFG) assert found[0]["test"] == "test_discount" assert found[0]["class"] == "tests.test_discount" assert "assert 0.0 == 90.0" in found[0]["message"] def test_an_assertion_defect_carries_no_path() -> None: """Naming the test file would invite weakening the test instead of fixing the cause.""" bundle = {"jenkins": {"failed_tests": [{"name": "t", "className": "c", "errorDetails": "boom"}]}} assert "path" not in module.extract_assertion_defects(bundle, CFG)[0] def test_assertion_instruction_forbids_weakening_the_test() -> None: bundle = {"jenkins": {"failed_tests": [{"name": "t", "className": "c", "errorDetails": "boom"}]}} text = module.defect_instruction(module.extract_assertion_defects(bundle, CFG)) assert "Fix the code under test" in text assert "Do not weaken, skip or delete" in text def test_malformed_failed_tests_are_skipped() -> None: bundle = {"jenkins": {"failed_tests": ["nope", {}, {"name": " "}, {"name": "ok"}]}} found = module.extract_assertion_defects(bundle, CFG) assert [f["test"] for f in found] == ["ok"] def test_extract_defects_combines_every_category() -> None: bundle = { "jenkins": { "console_failures": [ {"text": "ariadne/app.py:12:5: E501 Line too long\n" ' File "ariadne/app.py", line 42, in build\n' "NameError: name 'setttings' is not defined"} ], "console_tail": "", "failed_tests": [{"name": "t", "className": "c", "errorDetails": "boom"}], } } cats = {d["category"] for d in module.extract_defects(bundle, CFG)} assert cats == {module.LINT_VIOLATION, module.UNDEFINED_NAME, module.FAILING_ASSERTION} def test_name_and_assertion_extraction_never_raise_on_hostile_input() -> None: for bad in (None, 7, {"jenkins": {"console_failures": "no"}}, {"jenkins": {"failed_tests": 5}}): assert module.extract_name_defects(bad, CFG) == [] assert module.extract_assertion_defects(bad, CFG) == [] assert module.extract_name_defects(_bundle("NameError: name 'x' is not defined"), None) == [] def test_name_extraction_is_bounded() -> None: lines = [] for i in range(40): lines.append(f' File "ariadne/f{i}.py", line {i + 1}, in fn') lines.append(f"NameError: name 'sym{i}' is not defined") assert len(module.extract_name_defects(_bundle(*lines), CFG)) == module._MAX_DEFECTS def test_duplicate_name_defects_are_collapsed() -> None: lines = [' File "ariadne/app.py", line 4, in fn', "NameError: name 'zz' is not defined"] * 3 assert len(module.extract_name_defects(_bundle(*lines), CFG)) == 1 def test_a_name_defect_with_no_prefixes_configured_is_dropped() -> None: """An empty allowlist must deny, never permit everything.""" empty = {"allowed_path_prefixes": [], "allowed_suffixes": [".py"]} found = module.extract_name_defects( _bundle(' File "ariadne/app.py", line 1, in fn', "NameError: name 'q' is not defined"), empty ) assert found == [] def test_undefined_name_instruction_names_the_symbol() -> None: found = module.extract_name_defects( _bundle(' File "ariadne/app.py", line 42, in build', "NameError: name 'setttings' is not defined"), CFG, ) text = module.defect_instruction(found) assert "'setttings' is not defined or not importable" in text assert "ariadne/app.py line 42" in text def test_a_lint_defect_with_a_disallowed_suffix_is_dropped() -> None: """Suffix and prefix are separate gates; both must hold.""" cfg = {"allowed_path_prefixes": ["ariadne/"], "allowed_suffixes": [".go"]} assert module.extract_lint_defects(_bundle("ariadne/app.py:1:1: E501 Long"), cfg) == [] # --- the operator's control over what Hermes may be asked to fix --------------- def test_no_setting_means_every_category() -> None: """The default must be obvious rather than silently empty.""" assert module.enabled_categories({}) == module.ALL_CATEGORIES assert module.enabled_categories({"fix_categories": []}) == module.ALL_CATEGORIES assert module.enabled_categories(None) == module.ALL_CATEGORIES def test_a_category_can_be_switched_off() -> None: cfg = {**CFG, "fix_categories": [module.LINT_VIOLATION]} assert module.enabled_categories(cfg) == (module.LINT_VIOLATION,) bundle = { "jenkins": { "console_failures": [{"text": "ariadne/app.py:12:5: E501 Line too long"}], "console_tail": "", "failed_tests": [{"name": "t", "className": "c", "errorDetails": "boom"}], } } cats = {d["category"] for d in module.extract_defects(bundle, cfg)} assert cats == {module.LINT_VIOLATION} def test_an_unrecognised_category_is_ignored_not_trusted() -> None: cfg = {"fix_categories": ["lint_violation", "rewrite_everything"]} assert module.enabled_categories(cfg) == (module.LINT_VIOLATION,) def test_categories_keep_their_declared_order() -> None: cfg = {"fix_categories": [module.FAILING_ASSERTION, module.LINT_VIOLATION]} assert module.enabled_categories(cfg) == (module.LINT_VIOLATION, module.FAILING_ASSERTION)