titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

hermes: enforce per-file branch coverage in the quality gate

Browse Source 
The coverage check read only line-rate, so a file could pass with weak
branch coverage. Load both Cobertura rates and fail any tracked file
below the 95% floor on either metric, failing closed when branch
evidence is absent. Prove enforcement end to end with a synthetic
fully-lined but branch-weak file failing run_profile.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jenkins 2026-08-17 15:10:17 -03:00
parent 64272f52d2
commit f77119b238
5 changed files with 127 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
testing/quality_coverage.py
View File

@ -7,8 +7,8 @@ from pathlib import Path
from typing import Any
def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]:
"""Load per-file line-rate percentages from a Cobertura XML report."""
def _load_rates(xml_path: Path, root: Path) -> dict[str, dict[str, float | None]]:
"""Load per-file line and branch percentages from a Cobertura XML report."""
tree = ET.parse(xml_path)
xml_root = tree.getroot()
source_roots = [
@ -16,7 +16,7 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]:
for node in xml_root.findall("./sources/source")
if node.text
]
percentages: dict[str, float] = {}
rates: dict[str, dict[str, float | None]] = {}
for class_node in xml_root.findall(".//class"):
filename = class_node.attrib.get("filename")
line_rate = class_node.attrib.get("line-rate")
@ -32,33 +32,44 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]:
if candidate.exists():
key = candidate.relative_to(root).as_posix()
break
percentages[key] = float(line_rate) * 100.0
return percentages
branch_rate = class_node.attrib.get("branch-rate")
rates[key] = {
"line": float(line_rate) * 100.0,
"branch": None if branch_rate is None else float(branch_rate) * 100.0,
}
return rates
def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str]:
"""Return human-readable issues for tracked files below the coverage floor.
The report is intentionally per-file so a single weak module cannot hide
behind aggregate suite coverage.
Line and branch rates are both enforced per file so a single weak module
cannot hide behind aggregate suite coverage or line-only reporting.
"""
if not xml_path.exists():
return [f"coverage xml missing: {xml_path.relative_to(root)}"]
percentages = _load_percentages(xml_path, root)
rates = _load_rates(xml_path, root)
minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0))
issues: list[str] = []
for relative_path in contract.get("coverage", {}).get("tracked_files", []):
normalized = relative_path.replace("\\", "/")
percent = percentages.get(normalized)
if percent is None:
file_rates = rates.get(normalized)
if file_rates is None:
issues.append(f"coverage missing for tracked file: {relative_path}")
continue
if percent + 1e-9 < minimum:
issues.append(
f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)"
)
for metric in ("line", "branch"):
percent = file_rates[metric]
if percent is None:
issues.append(
f"{metric} coverage missing for tracked file: {relative_path}"
)
elif percent + 1e-9 < minimum:
issues.append(
f"{metric} coverage below {minimum:.1f}%: "
f"{relative_path} ({percent:.1f}%)"
)
return issues
@ -73,13 +84,13 @@ def compute_workspace_line_coverage(
if not xml_path.exists():
return 0.0
percentages = _load_percentages(xml_path, root)
rates = _load_rates(xml_path, root)
samples: list[float] = []
for relative_path in contract.get("coverage", {}).get("tracked_files", []):
normalized = relative_path.replace("\\", "/")
percent = percentages.get(normalized)
if percent is not None:
samples.append(percent)
file_rates = rates.get(normalized)
if file_rates is not None:
samples.append(file_rates["line"])
if not samples:
return 0.0
return round(sum(samples) / len(samples), 3)

2
testing/quality_gate.py
View File

@ -332,7 +332,7 @@ def run_profile(
results.append(
_result(
"coverage",
"Per-file 95% coverage floor for tracked quality-managed modules.",
"Per-file 95% line and branch coverage floor for tracked quality-managed modules.",
_status_from_issues(issues),
issues=issues,
coverage_xml=str(coverage_xml.relative_to(root)),

14
testing/tests/test_quality_contract.py
View File

@ -98,8 +98,9 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path):
<packages>
<package>
<classes>
<class filename="ok.py" line-rate="1.0" />
<class filename="low.py" line-rate="0.90" />
<class filename="ok.py" line-rate="1.0" branch-rate="1.0" />
<class filename="low.py" line-rate="0.90" branch-rate="1.0" />
<class filename="weak_branches.py" line-rate="1.0" branch-rate="0.90" />
</classes>
</package>
</packages>
@ -112,13 +113,14 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path):
contract = {
"coverage": {
"minimum_percent": 95.0,
"tracked_files": ["ok.py", "low.py", "missing.py"],
"tracked_files": ["ok.py", "low.py", "weak_branches.py", "missing.py"],
}
}
issues = run_coverage_check(contract, tmp_path, coverage_xml)
assert "coverage below 95.0%: low.py (90.0%)" in issues
assert "line coverage below 95.0%: low.py (90.0%)" in issues
assert "branch coverage below 95.0%: weak_branches.py (90.0%)" in issues
assert "coverage missing for tracked file: missing.py" in issues
@ -142,8 +144,8 @@ def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Pa
<packages>
<package>
<classes>
<class filename="mapped.py" line-rate="1.0" />
<class filename="{(tmp_path / 'absolute.py').as_posix()}" line-rate="1.0" />
<class filename="mapped.py" line-rate="1.0" branch-rate="1.0" />
<class filename="{(tmp_path / 'absolute.py').as_posix()}" line-rate="1.0" branch-rate="1.0" />
<class filename="skip.py" />
</classes>
</package>

65
testing/tests/test_quality_coverage_helpers.py
View File

@ -57,7 +57,7 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path:
<packages>
<package>
<classes>
<class filename="relative.py" line-rate="0.80" />
<class filename="relative.py" line-rate="0.80" branch-rate="1.0" />
</classes>
</package>
</packages>
@ -73,4 +73,65 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path:
coverage_xml,
)
assert issues == ["coverage below 95.0%: relative.py (80.0%)"]
assert issues == ["line coverage below 95.0%: relative.py (80.0%)"]
def test_run_check_fails_a_file_below_the_branch_floor(tmp_path: Path) -> None:
"""Full line coverage must never hide a file whose branch coverage is weak."""
coverage_xml = tmp_path / "coverage.xml"
coverage_xml.write_text(
textwrap.dedent(
"""\
<coverage>
<packages>
<package>
<classes>
<class filename="weak.py" line-rate="1.0" branch-rate="0.90" />
<class filename="solid.py" line-rate="1.0" branch-rate="0.95" />
</classes>
</package>
</packages>
</coverage>
"""
),
encoding="utf-8",
)
issues = run_check(
{"coverage": {"minimum_percent": 95.0, "tracked_files": ["weak.py", "solid.py"]}},
tmp_path,
coverage_xml,
)
assert issues == ["branch coverage below 95.0%: weak.py (90.0%)"]
def test_run_check_fails_closed_when_branch_evidence_is_absent(tmp_path: Path) -> None:
"""A report generated without branch measurement cannot satisfy the gate."""
coverage_xml = tmp_path / "coverage.xml"
coverage_xml.write_text(
textwrap.dedent(
"""\
<coverage>
<packages>
<package>
<classes>
<class filename="unmeasured.py" line-rate="1.0" />
</classes>
</package>
</packages>
</coverage>
"""
),
encoding="utf-8",
)
issues = run_check(
{"coverage": {"minimum_percent": 95.0, "tracked_files": ["unmeasured.py"]}},
tmp_path,
coverage_xml,
)
assert issues == ["branch coverage missing for tracked file: unmeasured.py"]

26
testing/tests/test_quality_gate.py
View File

@ -60,6 +60,32 @@ def test_run_profile_aggregates_internal_and_pytest_results(tmp_path: Path, monk
assert any(result.get("junit") == "build/junit-unit.xml" for result in summary["results"])
def test_run_profile_fails_synthetic_file_below_branch_floor(tmp_path: Path):
"""The gate itself must fail a tracked file with full lines but weak branches."""
build_dir = tmp_path / "build"
build_dir.mkdir()
(build_dir / "coverage-unit.xml").write_text(
"<coverage><packages><package><classes>"
'<class filename="weak.py" line-rate="1.0" branch-rate="0.90" />'
"</classes></package></packages></coverage>",
encoding="utf-8",
)
contract = {
"profiles": {"local": ["coverage"]},
"pytest_suites": {"unit": {"coverage_xml": "build/coverage-unit.xml"}},
"coverage": {"minimum_percent": 95.0, "tracked_files": ["weak.py"]},
}
summary = quality_gate.run_profile(contract, tmp_path, "local", build_dir)
assert summary["status"] == "failed"
coverage_result = summary["results"][0]
assert coverage_result["name"] == "coverage"
assert coverage_result["issues"] == ["branch coverage below 95.0%: weak.py (90.0%)"]
assert summary["workspace_line_coverage_percent"] == 100.0
def test_main_writes_summary_file(tmp_path: Path, monkeypatch):
summary = {"status": "ok", "profile": "local", "results": [], "manual_scripts": []}
monkeypatch.chdir(tmp_path)