81 lines
5.0 KiB
Python
81 lines
5.0 KiB
Python
"""Deterministic synthetic acceptance suites, never derived from roster data."""
|
|
from suite_contract import digest
|
|
|
|
PATTERNS = (
|
|
("frame", "Feed an encoded watchdog status frame to the message decoder",
|
|
"Capture the decoded object and compare fields, checksum status, and rejection code",
|
|
"Use a byte-array builder and a decoder-call fixture; no live hardware is required"),
|
|
("pulse", "Stop and resume physical watchdog pulses at the controller input",
|
|
"Measure reset-line timing with a digital capture fixture and check the deadline",
|
|
"Use a controllable pulse source and reset-line recorder; timing tolerance is supplied"),
|
|
("static", "Parse watchdog source-code analysis findings from an offline report",
|
|
"Count findings by severity and compare each rule identifier against the policy table",
|
|
"Load a static-analysis report parser and a severity policy fixture; do not execute firmware"),
|
|
("config", "Load a configuration document through the configuration parser",
|
|
"Assert accepted values or diagnostic positions using returned parser objects",
|
|
"Construct configuration text fixtures and call the parser without starting the network stack"),
|
|
("queue", "Drive producer and consumer tasks until the message queue reaches its limit",
|
|
"Observe queue depth, rejected writes, delivery ordering, and recovery after draining",
|
|
"Use concurrent task drivers, a bounded queue fixture, and sequence-number assertions"),
|
|
("access", "Submit role-scoped operations to the authorization decision function",
|
|
"Compare allow or deny decisions and audit-event fields against the permissions matrix",
|
|
"Create identity fixtures and an in-memory policy store; no interactive login is involved"),
|
|
)
|
|
|
|
|
|
def fixture(size):
|
|
"""Return interleaved cases and an independent expected implementation map."""
|
|
if size not in (14, 75, 363):
|
|
raise ValueError("unsupported synthetic size")
|
|
cases, expected = [], {}
|
|
for index in range(size):
|
|
family, action, assertion, setup = PATTERNS[index % len(PATTERNS)]
|
|
case = {"alias": f"CASE-{index + 1:04d}", "description": action + ". "
|
|
"Vary nominal, boundary, and malformed inputs while preserving the case objective. "
|
|
"The requirement reference supplies traceability only, not implementation instructions.",
|
|
"success_criteria": assertion + ". Preserve diagnostic evidence for this objective.",
|
|
"preconditions": setup + ". Reset fixture state between parameter variations.",
|
|
"operating_condition": "Qualified synthetic build; deterministic seed; isolated execution.",
|
|
"verification_method": "test" if family != "static" else "analysis",
|
|
"verifies": "[reference]"}
|
|
cases.append(case)
|
|
expected[case["alias"]] = family
|
|
# Same text at distant positions must retain two independent membership IDs.
|
|
cases[-2] = {**cases[1], "alias": cases[-2]["alias"]}
|
|
expected[cases[-2]["alias"]] = expected[cases[1]["alias"]]
|
|
specials = [(0, "thermal", "Cycle the thermal chamber and measure enclosure expansion",
|
|
"Expansion stays within the dimensional tolerance using a calibrated gauge"),
|
|
(size // 2, "acoustic", "Record acoustic output using an anechoic fixture",
|
|
"Spectral peak magnitude stays below the supplied frequency-dependent threshold"),
|
|
(size - 1, "reproducible", "Rebuild the same source twice in clean build containers",
|
|
"Compare artifact digests after removing only explicitly allowed timestamp metadata")]
|
|
for index, family, action, assertion in specials:
|
|
cases[index].update(description=action, success_criteria=assertion,
|
|
preconditions="Dedicated synthetic fixture; further procedure details are unknown.")
|
|
expected[cases[index]["alias"]] = family
|
|
return {"campaign": "SYNTHETIC", "suite": f"SUITE-{size}", "cases": cases}, expected
|
|
|
|
|
|
def allowed_synthetic(request):
|
|
"""Fail closed: external pilot permission covers only exact public fixtures."""
|
|
content = {k: request[k] for k in ("campaign", "suite", "cases")}
|
|
return digest(content) in {digest(fixture(n)[0]) for n in (14, 75, 363)}
|
|
|
|
|
|
def score(result, expected):
|
|
"""Measure alias coverage separately from pairwise implementation agreement."""
|
|
assigned = {alias: i for i, g in enumerate(result["groups"]) for alias in g["members"]}
|
|
aliases = list(expected)
|
|
tp = fp = fn = 0
|
|
for i, a in enumerate(aliases):
|
|
for b in aliases[i + 1:]:
|
|
actual = assigned.get(a, -1) == assigned.get(b, -2)
|
|
same = expected[a] == expected[b]
|
|
tp += actual and same
|
|
fp += actual and not same
|
|
fn += not actual and same
|
|
return {"coverage": set(assigned) == set(expected), "families": len(result["groups"]),
|
|
"pair_precision": tp / (tp + fp) if tp + fp else None,
|
|
"pair_recall": tp / (tp + fn) if tp + fn else None,
|
|
"false_merge_pairs": fp, "missed_merge_pairs": fn}
|