ariadne/tests/test_hermes_code_patch.py

254 lines
9.2 KiB
Python
Raw Normal View History

from __future__ import annotations
import json
import pytest
from ariadne.services import hermes_code_patch as module
INCIDENT_ID = "hermes-code-demo/7"
FILE_CONTENTS = "def discount(price):\n return price * 0.5\n"
def _patch_dict(**overrides) -> dict: # type: ignore[no-untyped-def]
base = {
"path": "src/discount.py",
"original": "return price * 0.5",
"replacement": "return price * 0.9",
"rationale": "restore the intended 10% discount",
}
base.update(overrides)
return base
def _payload(**overrides) -> dict: # type: ignore[no-untyped-def]
base = {
"incident_id": INCIDENT_ID,
"analysis": "The discount multiplier regressed from 0.9 to 0.5.",
"patch": _patch_dict(),
"human_required": False,
"reason": "small localized fix",
}
base.update(overrides)
return base
def _parse(payload: dict | None = None, raw: str | None = None, incident: str = INCIDENT_ID): # type: ignore[no-untyped-def]
text = raw if raw is not None else json.dumps(payload if payload is not None else _payload())
return module.parse_patch_response(text, incident)
def _proposed(**overrides) -> module.ProposedPatch: # type: ignore[no-untyped-def]
return module.ProposedPatch(**_patch_dict(**overrides))
def _cfg(**overrides) -> dict: # type: ignore[no-untyped-def]
base = {
"allowed_path_prefixes": ["src/"],
"allowed_suffixes": [".py"],
"max_patch_bytes": 4000,
"max_changed_lines": 20,
}
base.update(overrides)
return base
def test_parse_valid_response() -> None:
outcome = _parse()
assert outcome.valid is True
assert outcome.reject_reason is None
assert outcome.patch == module.ProposedPatch(
path="src/discount.py",
original="return price * 0.5",
replacement="return price * 0.9",
rationale="restore the intended 10% discount",
)
def test_parse_accepts_fenced_and_prose_wrapped_json() -> None:
raw = "Here you go.\n```json\n" + json.dumps(_payload()) + "\n```\nDone."
outcome = _parse(raw=raw)
assert outcome.valid is True
assert outcome.patch is not None
def test_parse_accepts_braces_and_escapes_inside_strings() -> None:
payload = _payload(analysis='Uses a {brace}, a "quoted" word, and a \\ escape.')
raw = "Prose with a stray } brace... " + json.dumps(payload) + ' Trailing {"not": "parsed"}'
outcome = _parse(raw=raw)
assert outcome.valid is True
assert outcome.patch is not None
def test_parse_rejects_missing_json() -> None:
assert _parse(raw="no json at all").reject_reason == "no_json_object_found"
assert _parse(raw="").reject_reason == "no_json_object_found"
def test_parse_rejects_invalid_json() -> None:
outcome = _parse(raw="{'incident_id': 'x'}")
assert outcome.valid is False
assert outcome.reject_reason.startswith("invalid_json:")
def test_parse_rejects_missing_keys() -> None:
payload = _payload()
payload.pop("analysis")
payload.pop("reason")
assert _parse(payload).reject_reason == "missing_keys: analysis, reason"
def test_parse_rejects_unexpected_keys() -> None:
assert _parse(_payload(extra=1)).reject_reason == "unexpected_keys: extra"
@pytest.mark.parametrize("field", ["incident_id", "analysis", "reason"])
def test_parse_rejects_non_string_scalars(field) -> None: # type: ignore[no-untyped-def]
outcome = _parse(_payload(**{field: 5}))
assert outcome.reject_reason == f"field_type_invalid: {field} must be a string"
def test_parse_rejects_non_boolean_human_required() -> None:
outcome = _parse(_payload(human_required="no"))
assert outcome.reject_reason == "field_type_invalid: human_required must be a boolean"
def test_parse_rejects_non_object_patch() -> None:
assert _parse(_payload(patch=[1])).reject_reason == "patch_invalid: must be an object or null"
assert _parse(_payload(patch="diff")).reject_reason == "patch_invalid: must be an object or null"
def test_parse_rejects_patch_with_wrong_keys() -> None:
missing = _patch_dict()
missing.pop("rationale")
expected = "patch_invalid: must have exactly path, original, replacement, rationale"
assert _parse(_payload(patch=missing)).reject_reason == expected
assert _parse(_payload(patch=_patch_dict(extra="x"))).reject_reason == expected
def test_parse_rejects_non_string_patch_fields() -> None:
outcome = _parse(_payload(patch=_patch_dict(original=7)))
assert outcome.reject_reason == "patch_invalid: fields must be strings"
@pytest.mark.parametrize("field", ["path", "original", "replacement"])
def test_parse_rejects_empty_patch_fields(field) -> None: # type: ignore[no-untyped-def]
outcome = _parse(_payload(patch=_patch_dict(**{field: ""})))
assert outcome.reject_reason == f"patch_field_empty: {field}"
def test_parse_rejects_incident_mismatch() -> None:
outcome = _parse(_payload(incident_id="other/1"))
assert outcome.reject_reason == f"incident_id_mismatch: got 'other/1' expected {INCIDENT_ID!r}"
def test_parse_rejects_human_required_true() -> None:
assert _parse(_payload(human_required=True)).reject_reason == "human_required"
def test_parse_rejects_null_patch() -> None:
assert _parse(_payload(patch=None)).reject_reason == "patch_missing"
def test_parse_analysis_extracts_string() -> None:
assert module.parse_analysis(json.dumps(_payload())).startswith("The discount multiplier")
assert module.parse_analysis("no json") == ""
assert module.parse_analysis("{'bad': json}") == ""
assert module.parse_analysis('{"analysis": 5}') == ""
assert module.parse_analysis("") == ""
def test_validate_patch_accepts_valid_patch() -> None:
assert module.validate_patch(_proposed(), _cfg(), FILE_CONTENTS) == (True, "valid")
@pytest.mark.parametrize(
"path",
[
"src/../secrets.py",
"/src/discount.py",
"src\\discount.py",
"src/disc\x00ount.py",
"",
],
)
def test_validate_patch_rejects_unsafe_paths(path) -> None: # type: ignore[no-untyped-def]
ok, reason = module.validate_patch(_proposed(path=path), _cfg(), FILE_CONTENTS)
assert (ok, reason) == (False, "path_unsafe")
def test_validate_patch_rejects_disallowed_prefix() -> None:
ok, reason = module.validate_patch(_proposed(path="lib/discount.py"), _cfg(), FILE_CONTENTS)
assert (ok, reason) == (False, "path_prefix_not_allowed")
def test_validate_patch_rejects_disallowed_suffix() -> None:
ok, reason = module.validate_patch(_proposed(path="src/discount.sh"), _cfg(), FILE_CONTENTS)
assert (ok, reason) == (False, "path_suffix_not_allowed")
def test_validate_patch_rejects_empty_allowlists() -> None:
cfg = _cfg(allowed_path_prefixes=[], allowed_suffixes=[])
ok, reason = module.validate_patch(_proposed(), cfg, FILE_CONTENTS)
assert (ok, reason) == (False, "path_prefix_not_allowed")
def test_validate_patch_treats_non_numeric_limits_as_zero() -> None:
cfg = _cfg(max_patch_bytes="lots", max_changed_lines=None)
ok, reason = module.validate_patch(_proposed(), cfg, FILE_CONTENTS)
assert (ok, reason) == (False, "patch_too_large")
def test_validate_patch_rejects_oversized_patch() -> None:
ok, reason = module.validate_patch(_proposed(), _cfg(max_patch_bytes=10), FILE_CONTENTS)
assert (ok, reason) == (False, "patch_too_large")
def test_validate_patch_rejects_too_many_changed_lines() -> None:
patch = _proposed(replacement="return (\n price\n * 0.9\n)")
ok, reason = module.validate_patch(patch, _cfg(max_changed_lines=2), FILE_CONTENTS)
assert (ok, reason) == (False, "too_many_changed_lines")
def test_validate_patch_counts_lines_from_larger_side() -> None:
patch = _proposed(original="return price * 0.5\n", replacement="return price * 0.9")
contents = FILE_CONTENTS
assert module.validate_patch(patch, _cfg(max_changed_lines=2), contents) == (True, "valid")
ok, reason = module.validate_patch(patch, _cfg(max_changed_lines=1), contents)
assert (ok, reason) == (False, "too_many_changed_lines")
def test_validate_patch_rejects_missing_original() -> None:
ok, reason = module.validate_patch(_proposed(original="return price * 0.75"), _cfg(), FILE_CONTENTS)
assert (ok, reason) == (False, "original_missing")
def test_validate_patch_rejects_ambiguous_original() -> None:
contents = FILE_CONTENTS + "\ndef other(price):\n return price * 0.5\n"
ok, reason = module.validate_patch(_proposed(), _cfg(), contents)
assert (ok, reason) == (False, "original_ambiguous")
def test_validate_patch_rejects_identical_replacement() -> None:
patch = _proposed(replacement="return price * 0.5")
ok, reason = module.validate_patch(patch, _cfg(), FILE_CONTENTS)
assert (ok, reason) == (False, "replacement_identical")
def test_apply_patch_replaces_single_occurrence() -> None:
patched = module.apply_patch(FILE_CONTENTS, _proposed())
assert patched == "def discount(price):\n return price * 0.9\n"
def test_apply_patch_raises_on_zero_occurrences() -> None:
with pytest.raises(ValueError, match="exactly once, found 0"):
module.apply_patch("nothing here", _proposed())
def test_apply_patch_raises_on_multiple_occurrences() -> None:
with pytest.raises(ValueError, match="exactly once, found 2"):
module.apply_patch(FILE_CONTENTS * 2, _proposed())