"""Deterministic scored mutation gate for Hermes handoff safety invariants.""" from __future__ import annotations import os import shutil import subprocess import sys import tempfile from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path @dataclass(frozen=True) class Mutation: """One source mutation and the isolated probe that must kill it.""" name: str path: str old: str new: str probe: str @dataclass(frozen=True) class MutationReport: """Scored result; any invalid, surviving, or baseline-broken case is an issue.""" total: int killed: int issues: tuple[str, ...] @property def score(self) -> float: return 100.0 if not self.total else round(self.killed * 100.0 / self.total, 2) MUTATIONS = ( Mutation( "policy-allows-git-fetch", "hermes_handoff_policy.py", ' "for-each-ref",\n', ' "fetch",\n "for-each-ref",\n', "from hermes_handoff_policy import *\n" "try: check_argv(('git','fetch','origin'))\n" "except PolicyError: raise SystemExit(0)\n" "raise SystemExit(1)\n", ), Mutation( "redaction-truncates-before-screening", "hermes_handoff_redaction.py", " return bound(redact(text), max_bytes)\n", " return bound(text, max_bytes)\n", "from hermes_handoff_redaction import safe_text\n" "assert 'A9' not in safe_text('password=A9secret-prefix', 23)[0]\n", ), Mutation( "empty-report-go", "hermes_handoff_model.py", " if (\n" " not self.results\n" " or self.invalid_statuses\n" " or self.harness_errors\n" " or self.blocking\n" " ):\n", " if (\n" " self.invalid_statuses\n" " or self.harness_errors\n" " or self.blocking\n" " ):\n", "from hermes_handoff_model import *\nassert Report('read-only','now').decision == NO_GO\n", ), Mutation( "generic-404-is-denial", "hermes_handoff_rules.py", "DENIAL_MARKERS = (\n", 'DENIAL_MARKERS = (\n "404 not found",\n', "from hermes_handoff_rules import looks_denied\n" "from hermes_handoff_exec import Outcome\n" "assert not looks_denied(Outcome(('x',),'x',1,'','404 not found'))\n", ), Mutation( "truncated-evidence-reaches-evaluator", "hermes_handoff_rules.py", " if truncated:\n" " return Evaluation(\n" " NOT_RUN, f\"truncated evidence from step(s): {', '.join(truncated)}\"\n" " )\n", " if False and truncated:\n return Evaluation(NOT_RUN, 'mutant')\n", "import hermes_handoff_evaluators\n" "import hermes_handoff_rules as r\n" "from hermes_handoff_exec import Outcome\n" "from hermes_handoff_model import *\n" "@r.evaluator('probe')\ndef probe(*_): return r.Evaluation(PASS,'bad')\n" "s=CheckSpec('x','x','x','probe')\n" "assert r.evaluate(s,{'x':Outcome(('x',),'x',0,truncated=True)}).status == NOT_RUN\n", ), Mutation( "truncated-outcome-is-ok", "hermes_handoff_exec.py", " return self.error is None and self.returncode == 0 and not self.truncated\n", " return self.error is None and self.returncode == 0\n", "from hermes_handoff_exec import Outcome\nassert not Outcome(('x',),'x',0,truncated=True).ok\n", ), Mutation( "nan-is-numeric", "hermes_handoff_json_rules.py", " or not math.isfinite(float(actual))\n", " or False\n", "import hermes_handoff_json_rules as j\n" "from hermes_handoff_model import *\nfrom hermes_handoff_exec import Outcome\n" "s=CheckSpec('x','x','x','json_numeric',expect={'step':'s','fields':{'x':{'min':0}}})\n" "j.parsed_step=lambda *_: ({'x':float('nan')},'s')\n" "assert j.evaluate_json_numeric(s,{'s':Outcome(('x',),'x',0)}).status == NOT_RUN\n", ), Mutation( "cleanup-ignores-branch-delete-failure", "hermes_handoff_ephemeral.py", ' if not deleted.ok:\n problems.append("ephemeral branch deletion failed")\n', ' if False and not deleted.ok:\n problems.append("mutant")\n', "import hermes_handoff_ephemeral as e\nfrom hermes_handoff_exec import Outcome\n" "class R:\n" " def __init__(self): self.i=0\n" " def run(self,*_a,**_k):\n" " self.i+=1\n" " return Outcome(('x',),'x',1 if self.i==1 else 0)\n" "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" "assert e._cleanup(R(),object(),q,'/api/v1/repos/atlas/titan-iac',[]).status == 'FAIL'\n", ), Mutation( "cleanup-accepts-open-pr", "hermes_handoff_ephemeral.py", ' elif "state does not match" not in problems:\n problems.append("state is not closed")\n', ' elif False:\n problems.append("mutant")\n', "import hermes_handoff_ephemeral as e\n" "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" "p={'number':1,'state':'open','draft':True,'merged':False,'base':{'ref':'main','sha':'2'*40},'head':{'ref':q.ref,'sha':'1'*40}}\n" "assert e._closed_pr_problems(p,q)\n", ), Mutation( "absence-check-passes-on-zero-evidence", "hermes_handoff_evaluators.py", " if not observed:\n" ' return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine")\n', " if False and not observed:\n return Evaluation(NOT_RUN, 'mutant')\n", "import hermes_handoff_evaluators as v\n" "from hermes_handoff_exec import Outcome\nfrom hermes_handoff_model import *\n" "s=CheckSpec('x','x','x','names_absent',steps=(Step('e','self',('kubectl',)),)," "expect={'step':'e','names':('GITEA_TOKEN',)})\n" "assert v.evaluate(s,{'e':Outcome(('x',),'x',0,'')}).status == NOT_RUN\n", ), Mutation( "repository-pin-is-a-bare-prefix", "hermes_handoff_policy.py", ' and not path.startswith((f"{repo_root}/", f"{repo_root}?"))\n', " and not path.startswith(repo_root)\n", "from hermes_handoff_policy import *\n" "try: check_argv((GITEA_CLIENT,'read','/api/v1/repos/atlas/titan-iac-evil/pulls'))\n" "except PolicyError: raise SystemExit(0)\n" "raise SystemExit(1)\n", ), Mutation( "exec-inner-command-is-not-rechecked", "hermes_handoff_policy.py", " check_argv(inner, mode)\n return\n", " return\n", "from hermes_handoff_policy import *\n" "argv=('kubectl','exec','pod','--','/usr/local/bin/kubectl','get','ns','-o','name'," "'--as','system:admin')\n" "try: check_argv(argv)\n" "except PolicyError: raise SystemExit(0)\n" "raise SystemExit(1)\n", ), Mutation( "unrecorded-step-loses-its-binary-attestation", "hermes_handoff_run.py", " executable_path=outcome.executable_path,\n" " executable_sha256=outcome.executable_sha256,\n", "", "import hermes_handoff_run as r\nfrom hermes_handoff_exec import Outcome\n" "from hermes_handoff_model import *\n" "class V:\n name='self'\n" "class N:\n" " def run(self,argv,_v,_b=None): return Outcome(argv,'self',0,'locked'," "executable_path='/usr/bin/git',executable_sha256='a'*64)\n" "s=CheckSpec('x','x','x','stdout_matches'," "steps=(Step('s','self',('git','status'),record=False),)," "expect={'step':'s','equals':'locked'})\n" "o=r.run_check(N(),s,{'self':V()}).outcomes[0]\n" "assert o.executable_sha256 == 'a'*64 and o.executable_path == '/usr/bin/git'\n", ), ) def _probe(root: Path, probe: str) -> subprocess.CompletedProcess[str]: environment = { "LC_ALL": "C", "PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONPATH": str(root), } return subprocess.run( [sys.executable, "-c", probe], stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=10, check=False, env=environment, ) def run_gate( repo_root: Path, mutations: Sequence[Mutation] = MUTATIONS ) -> MutationReport: """Run baseline and mutated probes in isolated copies and return the score.""" source_root = repo_root / "scripts" / "ops" issues: list[str] = [] killed = 0 with tempfile.TemporaryDirectory(prefix="hermes-mutation-") as temporary: clean = Path(temporary) / "clean" shutil.copytree(source_root, clean) for mutation in mutations: source = clean / mutation.path original = source.read_text(encoding="utf-8") if original.count(mutation.old) != 1: issues.append(f"{mutation.name}: mutation anchor count is not one") continue baseline = _probe(clean, mutation.probe) if baseline.returncode != 0: issues.append(f"{mutation.name}: baseline probe failed") continue mutated_text = original.replace(mutation.old, mutation.new, 1) try: compile(mutated_text, str(source), "exec") except SyntaxError: issues.append(f"{mutation.name}: mutant is syntactically invalid") continue mutant = Path(temporary) / f"mutant-{mutation.name}" shutil.copytree(clean, mutant) (mutant / mutation.path).write_text(mutated_text, encoding="utf-8") if _probe(mutant, mutation.probe).returncode != 0: killed += 1 else: issues.append(f"{mutation.name}: mutant survived") return MutationReport(len(mutations), killed, tuple(issues)) def main() -> int: """Run the repository mutation contract and print a bounded score.""" report = run_gate(Path(__file__).resolve().parents[1]) print( f"Hermes handoff mutation score: {report.killed}/{report.total} ({report.score:.2f}%)" ) for issue in report.issues: print(issue) return 0 if not report.issues and report.killed == report.total else 1 if __name__ == "__main__": # pragma: no cover raise SystemExit(main()) # pragma: no cover