d75d4d503e
The bounded patcher could only ever fix defects whose failure output names the source file. A pytest assertion that fails inside a test names only the test: on ariadne build 404 the defective file appeared zero times in the whole 174KB console, and candidate selection returned ariadne/app.py, the wrong file entirely. Admit test files as readable candidates and follow their absolute imports back to the module they exercise. Reading widens; writing does not. The patch validator gates on allowed_path_prefixes alone, so a test file can now be read for context and still never be patched - which also stops the classic bad fix of silencing a failing test instead of repairing the code. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
279 lines
10 KiB
Python
279 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from ariadne.services import hermes_code_candidates as module
|
|
|
|
|
|
def _cfg(**overrides) -> dict: # type: ignore[no-untyped-def]
|
|
base = {
|
|
"allowed_path_prefixes": ["src/", "tests/"],
|
|
"allowed_suffixes": [".py", ".rs", ".ts", ".tsx"],
|
|
"max_candidates": 3,
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _bundle(*regions: str, tail: str = "") -> dict:
|
|
return {
|
|
"jenkins": {
|
|
"console_failures": [
|
|
{"marker": "FAILED ", "line_number": index + 1, "text": text}
|
|
for index, text in enumerate(regions)
|
|
],
|
|
"console_tail": tail,
|
|
}
|
|
}
|
|
|
|
|
|
def _paths(text: str, **cfg_overrides) -> list[str]: # type: ignore[no-untyped-def]
|
|
return module.extract_candidate_paths(_bundle(text), _cfg(**cfg_overrides))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("line", "expected"),
|
|
[
|
|
(' File "src/ledger.py", line 12, in balance', "src/ledger.py"),
|
|
("src/ledger.py:14: AssertionError", "src/ledger.py"),
|
|
("FAILED tests/test_ledger.py::test_balance - assert 2 == 3", "tests/test_ledger.py"),
|
|
("ERROR tests/test_ledger.py", "tests/test_ledger.py"),
|
|
("thread 'main' panicked at src/lib.rs:12:34:", "src/lib.rs"),
|
|
(" --> src/lib.rs:20:5", "src/lib.rs"),
|
|
(" at src/client.ts:10:5", "src/client.ts"),
|
|
(" at Object.<anonymous> (src/panel.tsx:10:5)", "src/panel.tsx"),
|
|
("compiling src/build.rs:1:1 failed", "src/build.rs"),
|
|
],
|
|
)
|
|
def test_each_language_reference_pattern_is_matched(line: str, expected: str) -> None:
|
|
assert _paths(line) == [expected]
|
|
|
|
|
|
def test_named_patterns_are_reviewable() -> None:
|
|
names = [name for name, _pattern in module.FILE_REFERENCE_PATTERNS]
|
|
assert names == [
|
|
"python_traceback",
|
|
"pytest_summary",
|
|
"rust_diagnostic",
|
|
"js_stack_frame",
|
|
"path_with_line",
|
|
]
|
|
|
|
|
|
def test_workspace_prefix_is_stripped() -> None:
|
|
line = "/home/jenkins/agent/workspace/titan-api/src/ledger.py:14: AssertionError"
|
|
assert _paths(line) == ["src/ledger.py"]
|
|
|
|
|
|
def test_leading_dot_slash_and_duplicate_slashes_are_normalized() -> None:
|
|
assert _paths(".//src//ledger.py:14: boom") == ["src/ledger.py"]
|
|
|
|
|
|
def test_query_and_anchor_noise_is_dropped() -> None:
|
|
assert _paths("src/ledger.py?raw=1:14: boom") == ["src/ledger.py"]
|
|
assert _paths("src/ledger.py#L14:14: boom") == ["src/ledger.py"]
|
|
|
|
|
|
def test_suffix_outside_the_allowlist_is_rejected() -> None:
|
|
assert _paths("src/ledger.rb:14: boom") == []
|
|
assert _paths("src/ledger.rs:14: boom", allowed_suffixes=[".py"]) == []
|
|
|
|
|
|
def test_prefix_outside_the_allowlist_is_rejected() -> None:
|
|
assert _paths("vendor/ledger.py:14: boom") == []
|
|
assert _paths("vendor/helper.py:2: boom", allowed_path_prefixes=["src/"]) == []
|
|
|
|
|
|
def test_test_files_are_readable_outside_the_write_allowlist() -> None:
|
|
"""The failing test explains the defect and is often the only path named.
|
|
|
|
Reading it is what lets its imports lead back to the module under test.
|
|
Writing to it stays barred by the patch validator, which gates on
|
|
allowed_path_prefixes alone.
|
|
"""
|
|
|
|
assert _paths("tests/test_ledger.py:2: boom", allowed_path_prefixes=["src/"]) == [
|
|
"tests/test_ledger.py"
|
|
]
|
|
|
|
|
|
def test_empty_allowlists_accept_nothing() -> None:
|
|
assert _paths("src/ledger.py:14: boom", allowed_suffixes=[]) == []
|
|
assert _paths("src/ledger.py:14: boom", allowed_path_prefixes=[]) == []
|
|
|
|
|
|
def test_traversal_paths_are_rejected() -> None:
|
|
assert _paths("src/../../etc/shadow.py:1: boom") == []
|
|
|
|
|
|
def test_absolute_paths_are_rejected() -> None:
|
|
assert _paths("/etc/hosts.py:1: boom", allowed_path_prefixes=["/etc/", "src/"]) == []
|
|
|
|
|
|
def test_backslash_paths_never_become_candidates() -> None:
|
|
found = _paths("src\\ledger.py:12: boom", allowed_path_prefixes=["src/", "ledger.py"])
|
|
assert "src\\ledger.py" not in found
|
|
|
|
|
|
def test_nul_bearing_paths_are_rejected() -> None:
|
|
assert _paths("src/led\x00ger.py:1: boom") == []
|
|
|
|
|
|
def test_earliest_region_outranks_a_busier_later_region() -> None:
|
|
bundle = _bundle(
|
|
"src/first.py:3: AssertionError",
|
|
"src/second.py:4: boom\nsrc/second.py:5: boom\nsrc/second.py:6: boom",
|
|
)
|
|
assert module.extract_candidate_paths(bundle, _cfg()) == ["src/first.py", "src/second.py"]
|
|
|
|
|
|
def test_reference_count_breaks_ties_inside_one_region() -> None:
|
|
text = "src/one.py:1: boom\nsrc/many.py:2: boom\nsrc/many.py:3: boom"
|
|
assert _paths(text) == ["src/many.py", "src/one.py"]
|
|
|
|
|
|
def test_non_test_paths_rank_before_test_paths_at_equal_score() -> None:
|
|
text = "FAILED tests/test_ledger.py::test_balance\nsrc/ledger.py:2: AssertionError"
|
|
assert _paths(text) == ["src/ledger.py", "tests/test_ledger.py"]
|
|
|
|
|
|
def test_console_tail_is_scanned_after_the_regions() -> None:
|
|
bundle = _bundle("src/region.py:1: boom", tail="src/tail.py:9: boom")
|
|
assert module.extract_candidate_paths(bundle, _cfg()) == ["src/region.py", "src/tail.py"]
|
|
|
|
|
|
def test_tail_only_bundle_still_yields_candidates() -> None:
|
|
bundle = {"jenkins": {"console_failures": [], "console_tail": "src/tail.py:9: boom"}}
|
|
assert module.extract_candidate_paths(bundle, _cfg()) == ["src/tail.py"]
|
|
|
|
|
|
def test_repeated_references_are_deduped() -> None:
|
|
text = 'src/ledger.py:1: boom\nFile "src/ledger.py", line 1, in balance\nsrc/ledger.py:1: boom'
|
|
assert _paths(text) == ["src/ledger.py"]
|
|
|
|
|
|
def test_multiple_patterns_on_one_line_count_once() -> None:
|
|
text = " --> src/lib.rs:20:5\nsrc/other.rs:1: boom\nsrc/other.rs:2: boom"
|
|
assert _paths(text) == ["src/other.rs", "src/lib.rs"]
|
|
|
|
|
|
def test_max_candidates_caps_the_result() -> None:
|
|
text = "\n".join(f"src/file{index}.py:{index}: boom" for index in range(6))
|
|
assert len(_paths(text)) == 3
|
|
assert len(_paths(text, max_candidates=2)) == 2
|
|
|
|
|
|
def test_max_candidates_defaults_when_missing_or_unusable() -> None:
|
|
text = "\n".join(f"src/file{index}.py:{index}: boom" for index in range(6))
|
|
cfg = {"allowed_path_prefixes": ["src/"], "allowed_suffixes": [".py"]}
|
|
assert len(module.extract_candidate_paths(_bundle(text), cfg)) == module.DEFAULT_MAX_CANDIDATES
|
|
assert len(_paths(text, max_candidates=0)) == module.DEFAULT_MAX_CANDIDATES
|
|
assert len(_paths(text, max_candidates="nope")) == module.DEFAULT_MAX_CANDIDATES
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"bundle",
|
|
[
|
|
{},
|
|
{"jenkins": None},
|
|
{"jenkins": {}},
|
|
{"jenkins": {"console_failures": None, "console_tail": None}},
|
|
{"jenkins": {"console_failures": "not-a-list", "console_tail": 7}},
|
|
{"jenkins": {"console_failures": ["not-a-dict", {"text": None}, {}]}},
|
|
{"jenkins": {"console_failures": [{"text": "nothing to see"}]}},
|
|
],
|
|
)
|
|
def test_empty_or_malformed_bundles_yield_no_candidates(bundle) -> None: # type: ignore[no-untyped-def]
|
|
assert module.extract_candidate_paths(bundle, _cfg()) == []
|
|
|
|
|
|
@pytest.mark.parametrize("bundle", [None, "text", 7, [], {"jenkins": 5}])
|
|
def test_never_raises_on_hostile_input(bundle) -> None: # type: ignore[no-untyped-def]
|
|
assert module.extract_candidate_paths(bundle, _cfg()) == []
|
|
|
|
|
|
def test_never_raises_on_hostile_config() -> None:
|
|
bundle = _bundle("src/ledger.py:1: boom")
|
|
assert module.extract_candidate_paths(bundle, None) == []
|
|
assert module.extract_candidate_paths(bundle, {"allowed_suffixes": 5}) == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("path", "expected"),
|
|
[
|
|
("tests/test_ledger.py", True),
|
|
("src/test_helpers.py", True),
|
|
("src/ledger_test.py", True),
|
|
("tests/conftest.py", True),
|
|
("src/ledger.py", False),
|
|
("src/latest_run.py", False),
|
|
("src/contest.py", False),
|
|
],
|
|
)
|
|
def test_is_test_path_uses_default_markers(path: str, expected: bool) -> None:
|
|
assert module.is_test_path(path, {}) is expected
|
|
|
|
|
|
def test_is_test_path_honours_configured_markers() -> None:
|
|
cfg = {"test_path_markers": ("spec/",)}
|
|
assert module.is_test_path("spec/ledger_spec.py", cfg) is True
|
|
assert module.is_test_path("tests/test_ledger.py", cfg) is False
|
|
|
|
|
|
def test_is_test_path_never_raises() -> None:
|
|
assert module.is_test_path(None, {}) is False
|
|
assert module.is_test_path("src/a.py", {"test_path_markers": [None]}) is False
|
|
|
|
|
|
_CFG = {"allowed_suffixes": [".py"], "allowed_path_prefixes": ["ariadne/"], "max_candidates": 3}
|
|
|
|
|
|
def test_imports_lead_from_a_failing_test_to_the_module_under_test() -> None:
|
|
"""The defective source is often absent from the console entirely."""
|
|
|
|
source = "\n".join(
|
|
[
|
|
"from __future__ import annotations",
|
|
"import httpx",
|
|
"from ariadne.utils.errors import safe_error_detail",
|
|
"from ariadne.services import mailu",
|
|
]
|
|
)
|
|
paths = module.imported_source_paths(source, _CFG)
|
|
|
|
assert "ariadne/utils/errors.py" in paths
|
|
assert "ariadne/services/mailu.py" in paths
|
|
# Third-party and stdlib imports are outside the allowlist.
|
|
assert not any(path.startswith("httpx") for path in paths)
|
|
|
|
|
|
def test_imported_paths_stay_inside_the_write_allowlist() -> None:
|
|
"""Reading a test must not make test files patchable."""
|
|
|
|
source = "from tests.helpers import build_fixture\nfrom ariadne.app import create_app"
|
|
paths = module.imported_source_paths(source, _CFG)
|
|
|
|
assert "ariadne/app.py" in paths
|
|
assert not any(path.startswith("tests/") for path in paths)
|
|
|
|
|
|
def test_relative_imports_are_not_guessed() -> None:
|
|
"""A relative import gives no repository-root path to resolve."""
|
|
|
|
assert module.imported_source_paths("from . import errors", _CFG) == []
|
|
assert module.imported_source_paths("from .errors import safe", _CFG) == []
|
|
|
|
|
|
def test_queue_imported_sources_only_follows_tests() -> None:
|
|
"""A source file's imports are not chased; only the failing test's are."""
|
|
|
|
source = "from ariadne.utils.errors import safe_error_detail"
|
|
|
|
queue, queued = ["ariadne/app.py"], {"ariadne/app.py"}
|
|
module.queue_imported_sources(_CFG, "ariadne/app.py", source, queue, queued)
|
|
assert queue == ["ariadne/app.py"]
|
|
|
|
queue, queued = ["tests/test_utils.py"], {"tests/test_utils.py"}
|
|
module.queue_imported_sources(_CFG, "tests/test_utils.py", source, queue, queued)
|
|
assert "ariadne/utils/errors.py" in queue
|