atlas-iac/testing/tests/test_suite_multipass.py

"""Whole-suite orchestration, semantic review boundaries, and hard task sizing."""
import copy
import json
from pathlib import Path
import sys
import threading

import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'services/hermes/scripts'))
import suite_backends
import suite_multipass as workflow
from suite_contract import MODELS, Problem, encoded, preflight, validate_partition, validate_request, validate_result
from suite_jobs import Jobs
from suite_policy import invocation, validate_natural
from suite_assignments import decode, decision_keys
from suite_sizing import balanced_sizes, cap_families, disagreements
from suite_synthetic import fixture

CANARY = 'SYNTHETIC_CONTENT_NOT_FOR_DATABASE_OR_ROUTINE_LOGS'


def request(count):
    """Build fully synthetic coherent cases with separate aliases."""
    value = {'campaign': 'SYNTHETIC', 'suite': 'COHERENT', 'cases': [
        {'alias': f'CASE-{i:04d}', 'description': CANARY + ': inspect parser response.',
         'success_criteria': 'Call the JSON parser and assert returned fields against expected values.',
         'preconditions': 'Initialize an in-memory parser fixture with deterministic input.',
         'case_type': 'nominal' if i % 2 else 'fault injection'} for i in range(count)],
        'routing': {'allow_external': True, 'allowed_external_providers': ['claude']}}
    return validate_request(value, ['claude'])


def natural(source, parts, names=None):
    """Supply schema-valid mock engineering decisions and exact source support."""
    by_alias = {c['alias']: c for c in source['cases']}
    names = names or [f'Parser machinery {i}' for i in range(len(parts))]
    groups = []
    for name, members in zip(names, parts):
        groups.append({'name': name, 'description': 'Implement the shared parser fixture and returned-field assertions.',
            'members': list(members), 'common_work': 'Implement the shared parser fixture and returned-field assertions.',
            'rationale': CANARY + ': observations already exist; new expected values are inexpensive.',
            'uncertainty': '', 'variation_sets': [list(members)],
            'evidence': [{'alias': members[0], 'field': 'success_criteria',
                          'quote': by_alias[members[0]]['success_criteria'][:100]}]})
    return {'groups': groups}


def public(value):
    return {'groups': [{k: g[k] for k in ('name', 'description', 'members')} for g in value['groups']]}


def wire_value(value, call):
    """Encode fixed test decisions using the model's required per-alias representation."""
    assignments, groups = {}, []
    for index, group in enumerate(value['groups']):
        groups.append({k:v for k,v in group.items() if k not in {'members','variation_sets'}})
        variation = {a:i for i,part in enumerate(group.get('variation_sets',[])) for a in part}
        for alias in group['members']:
            assignments[alias] = {'family':index,'variation':variation[alias]} if variation else index
    result = {'groups':groups,'assignments':assignments}
    if 'decisions' in value:
        key_by_members = {frozenset(v):k for k,v in call['review_keys'].items()}
        result['decisions'] = {key_by_members[frozenset(d['source_members'])]:
                              {k:v for k,v in d.items() if k!='source_members'} for d in value['decisions']}
    return result


def review(value, original):
    """Attach one explicit keep/split decision for each original oversized family."""
    value = copy.deepcopy(value)
    value['decisions'] = []
    for old in original['groups']:
        if len(old['members']) > 5:
            descendants = [g for g in value['groups'] if set(g['members']) <= set(old['members'])]
            value['decisions'].append({'source_members': old['members'],
                'decision': 'split' if len(descendants) > 1 else 'keep',
                'rationale': 'Different machinery warrants splitting.' if len(descendants) > 1 else
                             'The same fixture and returned observations support all assertion variations.',
                'evidence': old['evidence']})
    return value


def install_backend(monkeypatch, source, reconciled, *, a=None, b=None, reviewed=None, audited=None, costs=None):
    """Capture complete invocations while substituting content-free usage metadata."""
    calls = []
    reviewed = review(reviewed or reconciled, reconciled)
    audited = review(audited or reviewed, reconciled)
    answers = {'proposal_a': a or public(reconciled), 'proposal_b': b or public(reconciled),
               'reconciliation': reconciled, 'large_family_review': reviewed, 'decision_audit': audited}
    def backend(value, cancel, *, invocation, progress=None):
        payload = json.loads(invocation['input'])
        assert {c['alias']: c for c in payload['suite']['cases']} == {c['alias']: c for c in source['cases']}
        assert payload['suite']['campaign'] == source['campaign']
        assert payload['suite']['suite'] == source['suite']
        calls.append((copy.deepcopy(value), copy.deepcopy(invocation)))
        if progress:
            progress({'cli_running': True, 'cli_output_bytes': 120, 'last_cli_activity_seconds_ago': 0})
        cost = costs[len(calls)-1] if costs else 0.1
        return wire_value(copy.deepcopy(answers[invocation['stage']]), invocation), {
            'model': MODELS['claude']['model'], 'usage': {'input_tokens': 100, 'output_tokens': 40},
            'cost_usd_estimate': cost, 'turns': 2, 'duration_api_ms': 10,
            'compaction': False, 'truncation': False, 'cli_diagnostics': {'exit_code': 0}}
    monkeypatch.setattr(suite_backends, 'claude_generate', backend)
    return calls


@pytest.mark.parametrize('count,sizes', [(6,[3,3]), (7,[4,3]), (11,[4,4,3]), (14,[5,5,4])])
def test_coherent_families_are_balanced_not_semantically_fragmented(monkeypatch, count, sizes):
    source = request(count)
    aliases = [c['alias'] for c in source['cases']]
    family = natural(source, [aliases], ['Reset recovery'])
    calls = install_backend(monkeypatch, source, family)
    result, metadata = workflow.generate(source, preflight(source), threading.Event(), '192.168.22.8')
    assert len(calls) == 5 and [len(g['members']) for g in result['groups']] == sizes
    assert metadata['natural_family_count'] == 1 and metadata['final_task_count'] == len(sizes)
    assert metadata['model_pass_count'] == 5 and metadata['turns'] == 10
    assert metadata['singleton_statistics'] == {'natural_singletons': 0, 'final_singletons': 0}
    assert [g['name'] for g in result['groups']] == [f'Reset recovery ({i}/{len(sizes)})' for i in range(1,len(sizes)+1)]
    assert all('Work-size part' in g['description'] for g in result['groups'])
    assert metadata['review_summary']['decision_audit'][0]['decision'] == 'keep'
    assert [call[0]['execution']['max_cost_usd'] for call in calls] == pytest.approx([30,29.9,29.8,29.7,29.6])
    assert all(calls[i+1][0]['execution']['max_seconds'] < calls[i][0]['execution']['max_seconds'] for i in range(4))
    validate_result(result, source)


def test_independent_orders_and_content_are_reproducible(monkeypatch):
    source = request(14)
    family = natural(source, [[c['alias'] for c in source['cases']]])
    calls = install_backend(monkeypatch, source, family)
    workflow.generate(source, preflight(source), threading.Event(), '192.168.22.8')
    a, b = (json.loads(calls[i][1]['input']) for i in (0,1))
    assert a['review_material'] == b['review_material'] == {}
    assert calls[0][1]['system'] == calls[1][1]['system']
    assert a['suite']['cases'] != b['suite']['cases']
    assert b['suite']['cases'] == workflow.ordered_request(source, True)['cases']
    assert all('members' not in c[1]['schema']['properties']['groups']['items']['properties'] for c in calls)
    assert all(set(c[1]['schema']['properties']['assignments']['required']) == {x['alias'] for x in source['cases']} for c in calls)
    assert 'proposal_a' in json.loads(calls[2][1]['input'])['review_material']


@pytest.mark.parametrize('count,effort', [(14, 'high'), (100, 'xhigh')])
def test_preflight_effort_is_preserved_through_all_five_passes(monkeypatch, count, effort):
    """Discovery, reconciliation, review and audit use the same selected effort."""
    source = request(count)
    family = natural(source, [[c['alias'] for c in source['cases']]])
    calls = install_backend(monkeypatch, source, family)
    selected = preflight(source)
    result, metadata = workflow.generate(source, selected, threading.Event(), '192.168.22.8')
    assert selected['reasoning'] == metadata['reasoning'] == effort
    assert len(calls) == 5
    assert all(call['reasoning'] == effort for _, call in calls)
    assert all(p['reasoning'] == effort for p in metadata['passes'])
    validate_result(result, source)


def test_progress_reports_activity_without_content_or_false_percentage(monkeypatch):
    source = request(7)
    family = natural(source, [[c['alias'] for c in source['cases']]])
    install_backend(monkeypatch, source, family)
    updates = []
    workflow.generate(source, preflight(source), threading.Event(), '192.168.22.8', updates.append)
    active = [value for value in updates if value.get('cli_running')]
    assert len(active) == 5
    assert [v['completed_model_passes'] for v in active] == [0,1,2,3,4]
    assert all(v['maximum_model_passes'] == 5 and v['cli_output_bytes'] == 120 for v in active)
    assert all(v['heartbeat_at'] > 0 and v['job_remaining_seconds'] <= 1800 for v in active)
    assert all(v['last_cli_activity_seconds_ago'] == 0 for v in active)
    assert updates[-1]['cli_running'] is False
    assert CANARY not in json.dumps(updates)


def test_thirty_minute_job_limit_retains_smaller_client_deadlines():
    source = request(7)
    assert source['execution']['max_seconds'] == 1800
    source['execution']['max_seconds'] = 900
    assert validate_request(source, ['claude'])['execution']['max_seconds'] == 900
    source['execution']['max_seconds'] = 1801
    with pytest.raises(Problem, match='invalid_timeout'):
        validate_request(source, ['claude'])


def test_cli_estimate_guard_default_and_client_override():
    source = request(7)
    assert source['execution']['max_cost_usd'] == 30
    source['execution']['max_cost_usd'] = 5
    assert validate_request(source, ['claude'])['execution']['max_cost_usd'] == 5
    source['execution']['max_cost_usd'] = 30.1
    with pytest.raises(Problem, match='invalid_cost_limit'):
        validate_request(source, ['claude'])


@pytest.mark.parametrize('size', [14,75,363])
def test_every_alias_is_required_in_internal_wire_schema(size):
    source = request(size)
    aliases = [c['alias'] for c in source['cases']]
    value = natural(source, [aliases])
    for stage in ('proposal_a','reconciliation','large_family_review','decision_audit'):
        context = {'original_partition':value} if stage=='decision_audit' else {'natural_partition':value}
        call = invocation(stage, source, None if stage=='proposal_a' else context)
        data = public(value) if stage=='proposal_a' else review(value,value) if stage in ('large_family_review','decision_audit') else value
        wire = wire_value(data, call)
        schema = call['schema']['properties']['assignments']
        assert set(schema['required']) == set(schema['properties']) == set(aliases)
        assert schema['additionalProperties'] is False
        assert decode(wire,call,source) == data


@pytest.mark.parametrize('mode,stage', [
    ('missing','assignment_keys'),('unknown','assignment_keys'),('wrong_index','assignment_index'),
    ('bool_index','assignment_index'),('unused_family','empty_assigned_group'),
    ('array_instead','assignment_keys'),('extra_group_field','assignment_group_fields')])
def test_invalid_wire_assignments_fail_without_partial_output_or_content(mode,stage):
    source = request(14)
    aliases = [c['alias'] for c in source['cases']]
    call = invocation('proposal_a',source)
    wire = wire_value(public(natural(source,[aliases])),call)
    if mode=='missing': del wire['assignments'][aliases[0]]
    elif mode=='unknown': wire['assignments'][CANARY]=0
    elif mode=='wrong_index': wire['assignments'][aliases[0]]=1
    elif mode=='bool_index': wire['assignments'][aliases[0]]=True
    elif mode=='unused_family': wire['groups'].append(dict(wire['groups'][0]))
    elif mode=='array_instead': wire['assignments']=[]
    else: wire['groups'][0]['members']=aliases
    with pytest.raises(Problem) as raised: decode(wire,call,source)
    assert raised.value.details['failure_stage']==stage
    assert CANARY not in json.dumps(raised.value.document())


def test_review_wire_keys_reference_exact_memberships_and_cannot_omit_a_decision():
    source=request(14)
    aliases=[c['alias'] for c in source['cases']]
    natural_value=natural(source,[aliases[:7],aliases[7:]])
    context={'original_partition':natural_value}
    call=invocation('decision_audit',source,context)
    keys=decision_keys(context)
    assert len(keys)==2 and set(call['schema']['properties']['decisions']['required'])==set(keys)
    wire=wire_value(review(natural_value,natural_value),call)
    del wire['decisions'][next(iter(keys))]
    with pytest.raises(Problem,match='incomplete_large_family_review'): decode(wire,call,source)


def test_real_semantic_subdivisions_precede_work_sizing(monkeypatch):
    source = request(11)
    aliases = [c['alias'] for c in source['cases']]
    for case in source['cases'][6:]:
        case['success_criteria'] = 'Capture reset-line waveform with an oscilloscope and measure transition duration.'
        case['preconditions'] = 'Configure pulse source and digital capture fixture.'
    original = natural(source, [aliases], ['Response observations'])
    split = natural(source, [aliases[:6],aliases[6:]], ['JSON response assertions','Waveform timing capture'])
    calls = install_backend(monkeypatch, source, original, reviewed=split, audited=split)
    result, metadata = workflow.generate(source, preflight(source), threading.Event(), '192.168.22.8')
    assert len(calls) == 5 and metadata['natural_family_count'] == 2
    assert sorted(len(g['members']) for g in result['groups']) == [3,3,5]
    assert metadata['review_summary']['decision_audit'][0]['decision'] == 'split'
    assert [d['family_name'] for d in metadata['review_summary']['capacity_divisions']] == ['JSON response assertions']


def test_bounded_audit_can_reverse_an_unjustified_semantic_split(monkeypatch):
    source = request(7)
    aliases = [c['alias'] for c in source['cases']]
    original = natural(source,[aliases],['Parser assertions'])
    split = natural(source,[aliases[:3],aliases[3:]],['Nominal parser checks','Rejection parser checks'])
    calls = install_backend(monkeypatch,source,original,reviewed=split,audited=original)
    result, metadata = workflow.generate(source,preflight(source),threading.Event(),'192.168.22.8')
    assert len(calls) == 5 and [len(g['members']) for g in result['groups']] == [4,3]
    assert metadata['review_summary']['large_family_review'][0]['decision'] == 'split'
    assert metadata['review_summary']['decision_audit'][0]['decision'] == 'keep'


def test_disagreement_is_not_resolved_by_transitive_closure(monkeypatch):
    source = request(3)
    aliases = [c['alias'] for c in source['cases']]
    a = public(natural(source,[aliases[:2],aliases[2:]],['Parser checks','Parser checks']))
    b = public(natural(source,[aliases[:1],aliases[1:]],['Other labels','Other labels']))
    final = natural(source,[aliases[:2],aliases[2:]],['Frame parser checks','Report parser checks'])
    final['groups'][1]['uncertainty'] = 'The observation interface remains unspecified.'
    calls = install_backend(monkeypatch,source,final,a=a,b=b)
    result, metadata = workflow.generate(source,preflight(source),threading.Event(),'192.168.22.8')
    assert len(calls) == 3 and len(result['groups']) == 2
    assert metadata['review_summary']['proposal_disagreements']['pair_count'] == 2
    assert metadata['review_summary']['unresolved_uncertainties']
    assert set(result['groups'][0]['members']) != set(aliases)


@pytest.mark.parametrize('count', [6,7,11,14,400])
def test_identical_text_distinct_aliases_stable_under_reordering(count):
    source = request(count)
    for case in source['cases']:
        case['case_type'] = 'nominal'
    aliases = [c['alias'] for c in source['cases']]
    family = natural(source,[aliases],['X'*56])
    first, _ = cap_families(family, source)
    source['cases'].reverse()
    family['groups'][0]['members'].reverse()
    family['groups'][0]['variation_sets'][0].reverse()
    second, _ = cap_families(family, source)
    assert first == second
    assert max(len(g['name']) for g in first['groups']) <= 64
    assert sorted(a for g in first['groups'] for a in g['members']) == sorted(aliases)
    sizes = [len(g['members']) for g in first['groups']]
    assert sizes == balanced_sizes(count) and max(sizes)-min(sizes) <= 1


@pytest.mark.parametrize('mutation,code', [('collision','duplicate_family_name'), ('long','invalid_json_result'),
    ('invented','invalid_case_assignments'), ('duplicate','invalid_case_assignments'), ('overcap','group_size_limit')])
def test_final_validation_is_independent_of_model_claims(mutation,code):
    source = request(6)
    aliases = [c['alias'] for c in source['cases']]
    value = public(natural(source,[aliases[:3],aliases[3:]],['Parser fixtures','Report fixtures']))
    if mutation == 'collision':
        value['groups'][1]['name'] = '  PARSER   fixtures '
    elif mutation == 'long': value['groups'][0]['name'] = 'x'*65
    elif mutation == 'invented': value['groups'][0]['members'][0] = 'CASE-NOT-SUPPLIED'
    elif mutation == 'duplicate': value['groups'][0]['members'].append(aliases[0])
    else: value = public(natural(source,[aliases]))
    with pytest.raises(Problem,match=code): validate_result(value,source)


def test_review_requires_every_oversized_family_and_real_source_support():
    source = request(14)
    aliases = [c['alias'] for c in source['cases']]
    original = natural(source,[aliases[:7],aliases[7:]])
    value = review(original,original)
    validate_natural(value,source,original['groups'])
    value['decisions'].pop()
    with pytest.raises(Problem,match='incomplete_large_family_review'):
        validate_natural(value,source,original['groups'])
    value = review(original,original)
    value['groups'][0]['evidence'][0]['quote'] = 'Invented equipment not in the source'
    with pytest.raises(Problem,match='invalid_review_evidence'):
        validate_natural(value,source,original['groups'])


@pytest.mark.parametrize('size', [14,75,363])
def test_existing_suite_sizes_have_separate_natural_and_task_counts(monkeypatch,size):
    source, expected = fixture(size)
    source['routing'] = {'allow_external':True,'allowed_external_providers':['claude']}
    source = validate_request(source,['claude'])
    buckets = {}
    for alias,family in expected.items(): buckets.setdefault(family,[]).append(alias)
    families = natural(source,list(buckets.values()),list(buckets))
    install_backend(monkeypatch,source,families)
    result, metadata = workflow.generate(source,preflight(source),threading.Event(),'192.168.22.8')
    assert metadata['natural_family_count'] == 9
    assert metadata['final_task_count'] == sum(len(balanced_sizes(len(p))) for p in buckets.values())
    assert all(1 <= len(g['members']) <= 5 for g in result['groups'])
    if size == 363: assert metadata['final_task_count'] > 7*metadata['natural_family_count']
    assert len(encoded({'result':result,**metadata})) < 1<<20


def test_whole_job_cost_budget_and_failure_no_partial_answer(monkeypatch):
    source = request(14)
    source['execution']['max_cost_usd'] = 5
    family = natural(source,[[c['alias'] for c in source['cases']]])
    calls = install_backend(monkeypatch,source,family,costs=[1,2,3])
    with pytest.raises(Problem,match='job_cost_budget_exhausted') as raised:
        workflow.generate(source,preflight(source),threading.Event(),'192.168.22.8')
    assert [c[0]['execution']['max_cost_usd'] for c in calls] == [5,4,2]
    assert raised.value.details['completed_model_passes'] == 3
    assert CANARY not in json.dumps(raised.value.document())


def test_expanded_reconciliation_capacity_checked_before_launch(monkeypatch):
    source = request(14)
    family = natural(source,[[c['alias'] for c in source['cases']]])
    calls = install_backend(monkeypatch,source,family)
    selected = preflight(source)
    original = workflow.capacity
    def capacity(call,*args):
        if call['stage'] == 'reconciliation':
            call['input'] += 'x'*(1<<20)
        return original(call,*args)
    monkeypatch.setattr(workflow,'capacity',capacity)
    with pytest.raises(Problem,match='pass_request_too_large'):
        workflow.generate(source,selected,threading.Event(),'192.168.22.8')
    assert len(calls) == 2


def test_local_only_fails_capacity_without_hosted_calls(monkeypatch):
    source = request(14)
    source['routing'] = {'allow_external':False,'allowed_external_providers':[]}
    monkeypatch.setattr(suite_backends,'claude_generate',lambda *a,**k: pytest.fail('hosted call'))
    with pytest.raises(Problem,match='capacity_or_unsupported_backend') as raised:
        preflight(source)
    assert set(raised.value.details['candidates']) == {'local'}


def test_review_is_authorized_result_only_and_never_persisted(tmp_path,monkeypatch,capsys):
    source = request(7)
    family = natural(source,[[c['alias'] for c in source['cases']]])
    install_backend(monkeypatch,source,family)
    monkeypatch.setattr(suite_backends,'switchyard_decision',lambda *_:None)
    jobs = Jobs(tmp_path/'jobs.sqlite')
    selected=preflight(source)
    job,_ = jobs.submit('owner','multi-pass-key',source,selected,'192.168.22.8',launch=False)
    jobs.run(job['job_id'],'owner',source,selected,'192.168.22.8')
    assert jobs.get(job['job_id'],'owner')['status'] == 'completed'
    assert 'review_summary' not in jobs.get(job['job_id'],'owner')
    assert CANARY in json.dumps(jobs.get(job['job_id'],'owner',result=True)['review_summary'])
    with pytest.raises(Problem,match='job_not_found'): jobs.get(job['job_id'],'other',result=True)
    assert CANARY not in jobs.db.execute('select document from jobs').fetchone()[0]
    assert CANARY not in capsys.readouterr().out
    replay,new = jobs.submit('owner','multi-pass-key',source,selected,'192.168.22.8',launch=False)
    assert not new and replay['job_id'] == job['job_id']
    jobs.results[job['job_id']] = (0,{}, {})
    with pytest.raises(Problem,match='result_expired_or_worker_restarted'):
        jobs.get(job['job_id'],'owner',result=True)


def test_shared_deadline_stops_after_prior_calls(monkeypatch):
    source = request(7)
    source['execution']['max_seconds'] = 60
    family = natural(source,[[c['alias'] for c in source['cases']]])
    calls = install_backend(monkeypatch,source,family)
    clock = [100.0]
    backend = suite_backends.claude_generate
    def advancing(*args,**kwargs):
        answer = backend(*args,**kwargs)
        clock[0] += 31
        return answer
    monkeypatch.setattr(workflow.time,'monotonic',lambda:clock[0])
    monkeypatch.setattr(suite_backends,'claude_generate',advancing)
    with pytest.raises(Problem,match='job_time_budget_exhausted'):
        workflow.generate(source,preflight(source),threading.Event(),'192.168.22.8')
    assert len(calls) == 2
    assert [c[0]['execution']['max_seconds'] for c in calls] == [60,29]


def test_missing_cost_measurement_stops_future_paid_calls(monkeypatch):
    source = request(7)
    family = natural(source,[[c['alias'] for c in source['cases']]])
    calls = install_backend(monkeypatch,source,family,costs=[None])
    with pytest.raises(Problem,match='budget_accounting_unavailable'):
        workflow.generate(source,preflight(source),threading.Event(),'192.168.22.8')
    assert len(calls) == 1


def test_cancellation_stops_between_model_passes(monkeypatch):
    source=request(7)
    family=natural(source,[[c['alias'] for c in source['cases']]])
    calls=install_backend(monkeypatch,source,family)
    cancel=threading.Event()
    backend=suite_backends.claude_generate
    def cancelling(*args,**kwargs):
        answer=backend(*args,**kwargs)
        cancel.set()
        return answer
    monkeypatch.setattr(suite_backends,'claude_generate',cancelling)
    with pytest.raises(Problem,match='cancelled'):
        workflow.generate(source,preflight(source),cancel,'192.168.22.8')
    assert len(calls) == 1


def test_review_cannot_cross_a_natural_boundary():
    source=request(14)
    aliases=[c['alias'] for c in source['cases']]
    original=natural(source,[aliases[:7],aliases[7:]])
    mixed=natural(source,[aliases[::2],aliases[1::2]])
    value=review(mixed,original)
    with pytest.raises(Problem): validate_natural(value,source,original['groups'])