rea/experiments/analysis-efficiency/review-model.py

"""Check selected model facts against public producer observations, not exit status."""

import argparse
import hashlib
import json
from pathlib import Path
import re


def read(path):
    return json.loads(path.read_text())


def strings(value):
    if isinstance(value, str):
        yield value
    elif isinstance(value, list):
        for entry in value:
            yield from strings(entry)
    elif isinstance(value, dict):
        for entry in value.values():
            yield from strings(entry)


def evidence_ids(value):
    return {s for s in strings(value) if re.fullmatch(r'ev_[a-f0-9]{64}', s)}


def decode(result):
    value = result.get('structured_content', result.get('structuredContent'))
    if value is not None:
        return value
    for entry in result.get('content', []):
        if entry.get('type') == 'text':
            try:
                return json.loads(entry['text'])
            except json.JSONDecodeError:
                pass
    return None


def named_scope(value, names):
    """Accept explicit equivalent scope labels; contradictory duplicates stay a failure."""
    if not isinstance(value, dict):
        return None
    selected = {name: value[name] for name in names if name in value}
    values = list(selected.values())
    if not values:
        return None
    if all(entry == values[0] for entry in values):
        return values[0]
    return {'conflicting_reported_scopes': selected}


def scope_with_verified_counts(value, expected_scope, returned_counts):
    """Validate explicit extra row counts; never erase unknown or conflicting facts."""
    if not isinstance(value, dict):
        return value, {}
    scope = dict(value)
    checks = {}
    for name, expected in returned_counts.items():
        if name in scope:
            reported = scope[name]
            checks[name] = type(reported) is int and reported == expected
            if name not in expected_scope:
                del scope[name]
    return scope, checks


def pcap_observation_fields(value):
    """Equivalent explicit labels are accepted; conflicting duplicates remain failures."""
    if not isinstance(value, dict):
        return value
    result = dict(value)
    for label, canonical in {
        'outer_limitations': 'limitations', 'normalized_kind': 'kind',
        'normalized_view': 'view', 'normalized_parent_evidence_id': 'parent_evidence_id',
        'normalized_coverage': 'coverage', 'normalized_unknowns': 'unknowns',
        'normalized_source_coverage': 'source_coverage',
    }.items():
        if label not in result:
            continue
        if canonical in result and json.dumps(result[label], sort_keys=True) != json.dumps(result[canonical], sort_keys=True):
            return {**result, 'conflicting_fields': [label, canonical]}
        result[canonical] = result.pop(label)
    return result


def pcap_complete_capture(value):
    """Project the selected metadata/counts from the cited complete receipt."""
    inventories = ['endpoints', 'dns_queries', 'tls_sni_values', 'protocol_distribution',
                   'http_request_uris', 'unattributed_dns_addresses']
    if not isinstance(value, dict) or not all(isinstance(value.get(key), list) for key in inventories):
        return None
    capture = {key: item for key, item in value.items() if key not in inventories + ['diagnostics']}
    capture['retained_counts'] = {key: len(value[key]) for key in inventories}
    diagnostics = value.get('diagnostics')
    if 'diagnostics' not in value:
        capture['diagnostics_code_units'] = None
    elif isinstance(diagnostics, dict) and all(isinstance(diagnostics.get(key + '_stderr'), str) for key in ['tshark', 'capinfos']):
        capture['diagnostics_code_units'] = {key: len(diagnostics[key + '_stderr'].encode('utf-16-le', errors='surrogatepass')) // 2
                                             for key in ['tshark', 'capinfos']}
    else:
        return None
    return capture


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--domain', choices=['wasm', 'evm', 'android', 'flutter-aot', 'sqlite', 'pcap', 'managed', 'managed-nonempty', 'firmware', 'inventory', 'javascript', 'native', 'browser'], required=True)
    parser.add_argument('--trial', type=Path, required=True)
    parser.add_argument('--reference', type=Path)
    parser.add_argument('--resource-log', type=Path, required=True)
    parser.add_argument('--output', type=Path, required=True)
    args = parser.parse_args()
    summary = read(args.trial / 'summary.json')
    settings = read(args.trial / 'trial-settings.json')
    answer_text = (args.trial / 'answer.txt').read_text().strip()
    fenced = answer_text.startswith('```json\n') and answer_text.endswith('\n```')
    answer = json.loads(answer_text[8:-4] if fenced else answer_text)
    reference = read(args.reference / 'answer.json') if args.reference else None
    observed_ids = set()
    producer_results = {}
    pcap_calls = []
    pcap_attempts = []
    workflow_checks = {'pcap_exact_call_sequence': False} if args.domain == 'pcap' else {}
    tool_errors = []
    with (args.trial / 'events.jsonl').open() as event_lines:
        for line in event_lines:
            event = json.loads(line)
            item = event.get('item', {})
            if event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call':
                continue
            result = item.get('result') or {}
            parsed = decode(result)
            if args.domain == 'pcap':
                pcap_attempts.append({'tool': item.get('tool'), 'arguments': item.get('arguments'),
                                      'status': item.get('status'),
                                      'evidence_id': parsed.get('evidence_id') if isinstance(parsed, dict) else None,
                                      'readable_evidence': isinstance(parsed, dict)
                                      and isinstance(parsed.get('normalized_result'), dict)
                                      and isinstance(parsed.get('evidence_id'), str)
                                      and re.fullmatch(r'ev_[a-f0-9]{64}', parsed['evidence_id']) is not None})
            if item['status'] == 'completed':
                # Only response text contributes observed IDs, never model arguments.
                for text in strings(result):
                    observed_ids.update(re.findall(r'ev_[a-f0-9]{64}', text))
                if parsed is not None:
                    if args.domain == 'pcap':
                        pcap_calls.append((item['tool'], item.get('arguments'), parsed))
                    else:
                        producer_results.setdefault(item['tool'], []).append(parsed)
            else:
                tool_errors.append({'tool': item['tool'], 'arguments': item['arguments'], 'response': parsed if parsed is not None else result})
                error = parsed.get('error') if isinstance(parsed, dict) else None
                if isinstance(error, dict) and error.get('code') == 'resource_constraint':
                    details = error.get('details')
                    limits = details.get('reported_limits') if isinstance(details, dict) else None
                    if isinstance(limits, dict):
                        observed_ids.update(evidence_ids(limits.get('evidence_reference')))
    reported_ids = evidence_ids(answer.get('evidence_ids'))
    checks = {
        'reported_evidence_ids_observed_in_responses': bool(reported_ids) and reported_ids <= observed_ids,
        'temporary_profile_removed': read(args.trial / 'cleanup.json')['profile_removed'],
    }
    if summary.get('selected_artifact_sha256_before') is not None:
        checks['selected_artifact_bytes_unchanged'] = summary['selected_artifact_bytes_unchanged']
    if reference is not None:
        checks['target_digest'] = answer.get('artifact_sha256') == reference['artifact_sha256']
    differences = {}

    def check(name, actual, expected):
        checks[name] = actual == expected
        if not checks[name]:
            differences[name] = {'answer': actual, 'producer': expected}

    domain = args.domain
    if domain == 'wasm':
        for key in ['validation', 'section_count', 'import_count', 'export_count', 'exports_first4', 'wat_sha256', 'wat_code_units', 'runtime_execution', 'network_fetch']:
            check(key, answer.get(key), reference[key])
        for key in ['release', 'source_commit']:
            check('wabt_' + key, answer.get('wabt_' + key), reference['tool_profile'][key])
    elif domain == 'evm':
        n = producer_results['inspect_evm_interface'][-1]['normalized_result']
        if n.get('kind') == 'summary':
            check('evm_summary.parent_operation', n.get('parent_operation'), 'inspect_evm_interface')
            checks['evm_summary.parent_reference_observed'] = n.get('parent_evidence_id') in observed_ids
            n = n['summary']
        for key, expected in {'artifact_bytes': n['artifact']['bytes'], 'bytecode_sha256': n['bytecode']['sha256'], 'bytecode_bytes': n['bytecode']['bytes'], 'functions': n['functions'], 'evidence_kind': n['evidence_kind'], 'discovery_completeness': n['discovery_completeness'], 'runtime_execution': n['runtime_execution'], 'bytecode_kind': n['bytecode']['kind'], 'hardfork': n['bytecode']['hardfork'], 'deployment_authenticity': n['bytecode']['deployment_authenticity']}.items():
            check(key, answer.get(key), expected)
        check('evm_original_limitations', answer.get('limitations'), n['limitations'])
    elif domain == 'sqlite':
        for key in ['artifact', 'wal', 'engine', 'database_encoding', 'schema', 'rows', 'producer_limitations', 'limitations']:
            # Missing is not SQL/JSON null; booleans are not numeric counters.
            check(key, (key in answer, json.dumps(answer.get(key), sort_keys=True)),
                  (True, json.dumps(reference[key], sort_keys=True)))
    elif domain == 'pcap':
        check('capture', ('capture' in answer, json.dumps(answer.get('capture'), sort_keys=True)),
              (True, json.dumps(reference['capture'], sort_keys=True)))
        prefix = answer.get('http_request_uris_first2')
        records = {}
        for tool, arguments, value in pcap_calls:
            if not isinstance(value, dict) or not isinstance(value.get('evidence_id'), str):
                continue
            identity = value['evidence_id']
            normalized = value.get('normalized_result')
            supported = (isinstance(normalized, dict) and isinstance(value.get('limitations'), list)
                         and all(isinstance(item, str) for item in value['limitations']))
            if supported:
                supported = all(key not in normalized or (isinstance(normalized[key], list)
                                and all(isinstance(item, str) for item in normalized[key]))
                                for key in ['limitations', 'unknowns'])
            if supported and ('summary' in normalized or normalized.get('kind') == 'summary'):
                supported = isinstance(normalized.get('summary'), dict)
            if supported and ('page' in normalized or normalized.get('kind') == 'pcap'):
                supported = isinstance(normalized.get('page'), dict)
            entry = (tool, arguments, value) if supported else None
            if identity not in records:
                records[identity] = entry
            elif json.dumps(records[identity], sort_keys=True) != json.dumps(entry, sort_keys=True):
                # An ID with conflicting producers or envelopes is ambiguous.
                records[identity] = None
        observations = answer.get('delivery_observations')
        canonical_observations = [pcap_observation_fields(value) for value in observations] if isinstance(observations, list) else observations
        if isinstance(prefix, dict):
            checks['pcap_prefix_fields_supported'] = set(prefix) <= {'items', 'coverage', 'source_coverage'} and 'items' in prefix
            if 'source_coverage' in prefix:
                check('pcap_prefix_source_coverage', prefix['source_coverage'], reference['capture']['coverage'])
            if 'coverage' in prefix:
                last = canonical_observations[-1] if isinstance(canonical_observations, list) and canonical_observations else None
                identity = last.get('evidence_id') if isinstance(last, dict) else None
                entry = records.get(identity) if isinstance(identity, str) else None
                record = entry[2] if entry is not None else None
                page = record['normalized_result'] if record is not None else {}
                checks['pcap_prefix_coverage_observed'] = (
                    page.get('kind') == 'pcap'
                    and page.get('page', {}).get('facet') == 'http-uris'
                    and json.dumps(page['page'].get('items'), sort_keys=True) == json.dumps(prefix.get('items'), sort_keys=True)
                    and 'coverage' in page
                    and json.dumps(prefix['coverage'], sort_keys=True) == json.dumps(page['coverage'], sort_keys=True)
                )
            prefix = prefix.get('items')
        check('http_request_uris_first2', ('http_request_uris_first2' in answer, json.dumps(prefix, sort_keys=True)),
              (True, json.dumps(reference['http_request_uris_first2'], sort_keys=True)))
        expected_observations = []
        if isinstance(canonical_observations, list):
            for observation in canonical_observations:
                identity = observation.get('evidence_id') if isinstance(observation, dict) else None
                entry = records.get(identity) if isinstance(identity, str) else None
                record = entry[2] if entry is not None else None
                if record is None:
                    expected_observations.append(None)
                    continue
                value = record['normalized_result']
                expected = {'evidence_id': record['evidence_id'], 'limitations': record['limitations']}
                for key in ['kind', 'view', 'parent_evidence_id', 'coverage', 'unknowns']:
                    if key in value:
                        expected[key] = value[key]
                if 'limitations' in value:
                    expected['normalized_limitations'] = value['limitations']
                if 'source_coverage' in observation and 'source_coverage' in value:
                    expected['source_coverage'] = value['source_coverage']
                expected_observations.append(expected)
        checks['pcap_delivery_observations_nonempty'] = isinstance(observations, list) and bool(observations)
        check('pcap_original_delivery_observations', json.dumps(canonical_observations, sort_keys=True),
              json.dumps(expected_observations, sort_keys=True))
        if isinstance(canonical_observations, list) and canonical_observations and all(isinstance(x, dict) for x in canonical_observations):
            check('pcap_requested_delivery_chain', [x.get('kind', 'complete') for x in canonical_observations],
                  [x.get('kind', 'complete') for x in reference['delivery_observations']])
            check('pcap_requested_views', json.dumps([x.get('view') for x in canonical_observations], sort_keys=True),
                  json.dumps([x.get('view') for x in reference['delivery_observations']], sort_keys=True))
            if len(reference['delivery_observations']) == 2:
                check('pcap_summary_coverage', json.dumps(canonical_observations[0].get('coverage'), sort_keys=True),
                      json.dumps({'status': 'complete-within-view', 'examined': 1, 'total': 1,
                                  'next_offset': None, 'exhausted': True}, sort_keys=True))
            identities = [x.get('evidence_id') for x in canonical_observations]
            checks['pcap_observations_have_reported_ids'] = (
                isinstance(answer.get('evidence_ids'), list)
                and answer['evidence_ids'] == identities
                and all(isinstance(value, str) and re.fullmatch(r'ev_[a-f0-9]{64}', value) for value in identities)
                and len(set(identities)) == len(identities)
            )
            entries = [records.get(identity) if isinstance(identity, str) else None for identity in identities]
            cited = [entry[2] if entry is not None else None for entry in entries]
            expected_tools = ['inspect_pcap_capture'] + (['inspect_analysis_view'] if len(reference['delivery_observations']) == 2 else [])
            check('pcap_cited_producer_tools', [entry[0] if entry is not None else None for entry in entries], expected_tools)
            # Summary delivery delegates to a retained view; its envelope operation
            # is inspect_analysis_view even though the requested tool is inspect_pcap_capture.
            expected_operations = ['inspect_pcap_capture'] if len(expected_tools) == 1 else ['inspect_analysis_view'] * 2
            check('pcap_cited_operations', [record.get('operation') if isinstance(record, dict) else None for record in cited], expected_operations)
            path = reference['capture']['target']['path']
            expected_arguments = [{'path': path}]
            expected_parameters = [{'path': path}]
            if len(expected_tools) == 2:
                parent = cited[0]['normalized_result'].get('parent_evidence_id') if isinstance(cited[0], dict) else None
                views = [observation.get('view') for observation in reference['delivery_observations']]
                expected_arguments = [{'path': path, 'detail': 'summary'},
                                      {'source': {'kind': 'retained-evidence', 'evidence_id': parent}, 'view': views[-1]}]
                expected_parameters = [{'parent_evidence_id': parent, 'view': view} for view in views]
            check('pcap_cited_call_arguments', json.dumps([entry[1] if entry is not None else None for entry in entries], sort_keys=True),
                  json.dumps(expected_arguments, sort_keys=True))
            workflow_checks['pcap_exact_call_sequence'] = (
                len(identities) == len(expected_tools)
                and json.dumps(pcap_attempts, sort_keys=True) == json.dumps([
                    {'tool': tool, 'arguments': arguments, 'status': 'completed',
                     'evidence_id': identity, 'readable_evidence': True}
                    for tool, arguments, identity in zip(expected_tools, expected_arguments, identities)
                ], sort_keys=True)
            )
            check('pcap_cited_parameters', json.dumps([record.get('parameters') if isinstance(record, dict) else None for record in cited], sort_keys=True),
                  json.dumps(expected_parameters, sort_keys=True))
            checks['pcap_cited_subjects_match_artifact'] = bool(cited) and all(
                isinstance(record, dict)
                and isinstance(record.get('subject'), dict)
                and isinstance(record['subject'].get('digest'), dict)
                and record['subject']['digest'].get('sha256') == reference['artifact_sha256']
                and record['subject'].get('local_path') == reference['capture']['target']['path']
                for record in cited
            )
            checks['pcap_cited_parents_linked'] = bool(cited) and all(isinstance(record, dict) for record in cited) and (
                len(cited) == 1 or (len(cited) == 2 and all(
                    isinstance(record['normalized_result'].get('parent_evidence_id'), str)
                    and re.fullmatch(r'ev_[a-f0-9]{64}', record['normalized_result']['parent_evidence_id'])
                    and record['normalized_result'].get('parent_digest') == record['normalized_result']['parent_evidence_id'][3:]
                    and record['normalized_result']['parent_evidence_id'] != record['evidence_id']
                    and isinstance(record.get('evidence_links'), list)
                    and record['normalized_result']['parent_evidence_id'] in record['evidence_links']
                    for record in cited
                ))
            )
            checks['pcap_cited_source_supports_uri_prefix'] = False
            checks['pcap_cited_capture'] = False
            if all(isinstance(record, dict) for record in cited):
                if len(cited) == 1:
                    source = cited[0]['normalized_result']
                    check('pcap_cited_capture', json.dumps(answer.get('capture'), sort_keys=True),
                          json.dumps(pcap_complete_capture(source), sort_keys=True))
                    checks['pcap_cited_source_supports_uri_prefix'] = (
                        source.get('target') == reference['capture']['target']
                        and isinstance(source.get('http_request_uris'), list)
                        and json.dumps(source['http_request_uris'][:2], sort_keys=True) == json.dumps(prefix, sort_keys=True)
                    )
                elif len(cited) == 2:
                    source, page = [record['normalized_result'] for record in cited]
                    capture = source.get('summary', {})
                    check('pcap_cited_capture', json.dumps(answer.get('capture'), sort_keys=True),
                          json.dumps(capture, sort_keys=True))
                    selected_page = page.get('page', {})
                    checks['pcap_cited_source_supports_uri_prefix'] = (
                        source.get('kind') == 'summary'
                        and source.get('parent_operation') == 'inspect_pcap_capture'
                        and capture == reference['capture']
                        and source.get('artifact') == {key: capture['target'][key] for key in ['path', 'sha256']}
                        and isinstance(source.get('parent_evidence_id'), str)
                        and page.get('parent_evidence_id') == source['parent_evidence_id']
                        and page.get('parent_operation') == 'inspect_pcap_capture'
                        and json.dumps(page.get('view'), sort_keys=True) == json.dumps({'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}, sort_keys=True)
                        and selected_page.get('facet') == 'http-uris'
                        and json.dumps(selected_page.get('items'), sort_keys=True) == json.dumps(prefix, sort_keys=True)
                        and page.get('source_coverage') == capture.get('coverage')
                        and json.dumps(page.get('coverage'), sort_keys=True) == json.dumps(reference['delivery_observations'][-1].get('coverage'), sort_keys=True)
                        and page.get('artifact') == {key: capture['target'][key] for key in ['path', 'sha256']}
                    )
        else:
            checks['pcap_requested_delivery_chain'] = False
            if len(reference['delivery_observations']) == 2:
                checks['pcap_summary_coverage'] = False
            checks['pcap_observations_have_reported_ids'] = False
            checks['pcap_requested_views'] = False
            checks['pcap_cited_source_supports_uri_prefix'] = False
            checks['pcap_cited_capture'] = False
            checks['pcap_cited_subjects_match_artifact'] = False
            checks['pcap_cited_parents_linked'] = False
            checks['pcap_cited_producer_tools'] = False
            checks['pcap_cited_operations'] = False
            checks['pcap_cited_call_arguments'] = False
            checks['pcap_cited_parameters'] = False
    elif domain == 'flutter-aot':
        for key in ['target', 'abi', 'libapp', 'string_pool', 'coverage', 'limitations']:
            check(key, answer.get(key), reference[key])
    elif domain == 'android':
        for key in ['package_name', 'version_name', 'version_code', 'matches']:
            check(key, answer.get(key), reference[key])
        fields = ['reported_signature', 'overload_index', 'dex_descriptor']
        checks['onCreate_fields_present_including_explicit_null'] = all(all(k in row for k in fields) for row in answer.get('onCreate', []))
        check('all_onCreate_descriptors', [{k: row.get(k) for k in fields} for row in answer.get('onCreate', [])], [{k: row[k] for k in fields} for row in reference['onCreate']])
    elif domain == 'managed':
        n = reference['artifact']
        for group, fields in [('assembly', ['name', 'version']), ('module', ['name', 'mvid'])]:
            for field in fields:
                check(group + '.' + field, answer.get(group, {}).get(field), n[group][field])
        for key in ['classification', 'target_frameworks']:
            check(key, answer.get(key), n[key])
        for key in ['module_refs', 'pinvoke_imports']:
            check(key, answer.get('native_boundaries', {}).get(key), reference['boundaries'][key])
        check('artifact_coverage', named_scope(answer.get('coverage'), ['artifact', 'managed_artifact', 'managed_metadata']), n['coverage'])
        boundary_scope, counts = scope_with_verified_counts(
            named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']),
            reference['boundaries']['coverage'],
            {'module_ref_count': len(reference['boundaries']['module_refs']),
             'pinvoke_import_count': len(reference['boundaries']['pinvoke_imports'])})
        check('native_boundaries_coverage', boundary_scope, reference['boundaries']['coverage'])
        checks.update({'native_coverage_' + name: valid for name, valid in counts.items()})
    elif domain == 'managed-nonempty':
        n = reference['boundaries']
        check('module_mvid', answer.get('module_mvid'), n['module']['mvid'])
        check('summary', {key: answer.get('summary', {}).get(key) for key in n['summary']}, n['summary'])
        check('all_original_module_refs', answer.get('module_refs'), n['module_refs'])
        check('first3_complete_pinvoke_records', answer.get('pinvoke_first3'), n['pinvoke_imports'][:3])
        check('coverage', answer.get('coverage'), n['coverage'])
    elif domain == 'firmware':
        source = producer_results['inspect_firmware_regions'][-1]
        n = source['normalized_result']
        result = answer['result']
        check('input_size', result.get('input_size'), n['input_size'])
        check('examined_coverage', result.get('coverage', result.get('examined_coverage')), n['coverage'])
        fields = ['provider_id', 'offset', 'reported_size', 'signature', 'description', 'reported_confidence', 'size_basis']
        check('all_original_region_records', [{k: row.get(k) for k in fields} for row in result.get('regions', [])], [{k: row[k] for k in fields} for row in n['regions']])
        engine = result.get('engine', result.get('provider_provenance', {}).get('engine', {}))
        for key in ['name', 'version']:
            check('engine.' + key, engine.get(key), n['engine'][key])
        check('engine_executable_digest', engine.get('executable_sha256', engine.get('artifact_sha256')), n['engine']['executable_sha256'])
    elif domain == 'inventory':
        for key in ['manifest', 'entry_kind_counts', 'integrity_contradictions']:
            check(key, answer.get(key), reference[key])
        original = producer_results['inspect_artifact'][-1]['normalized_result']
        if original.get('kind') == 'summary':
            branches = original['summary']['inspection']['unexplored_branches']
        else:
            branches = original['unexplored_branches']
        # Caller-selected integrity policy participates in atomic Evidence and
        # branch identity. Match those IDs to this original response, not a
        # reference invocation that may use a different policy.
        check('unexplored_branches_with_original_ids', answer.get('unexplored_branches'), branches)
        coverage = answer.get('coverage', {})
        check('inspection_coverage', coverage.get('inspection', coverage), reference['coverage'])
        paths = answer.get('requested_paths')
        if isinstance(paths, dict):
            paths = [row for value in paths.values() for row in (value if isinstance(value, list) else [value]) if row is not None]
        canonical = lambda row: json.dumps(row, sort_keys=True, separators=(',', ':'))
        check('original_selected_path_records', sorted(map(canonical, paths or [])), sorted(map(canonical, reference['requested_paths'])))
    elif domain == 'javascript':
        for key in ['input_path', 'source_selection', 'statistics', 'electron']:
            check(key, answer.get(key), reference[key])
        coverage = answer['coverage']
        expected = reference['coverage']
        for key in ['application_unknowns', 'semantic_unknowns']:
            check('coverage.' + key, coverage.get(key), expected[key])
        for layer in ['application', 'semantic']:
            for key, value in expected[layer].items():
                if key in ['limits', 'families']: continue
                check('coverage.' + layer + '.' + key, coverage[layer].get(key), value)
        # Budgets vary with parent heap and belong to this actual invocation.
        # Exact optional provenance remains a separate strict check; omissions
        # must not be hidden by accepting the coarse coverage summary.
        original = producer_results['analyze_javascript_application'][-1]['normalized_result']['summary']['coverage']
        check('full_coverage_provenance', coverage, original)
        # The question names types/digests, not literal answer property names.
        # Exact node IDs select the producer's main.js records. No source or
        # Evidence values are rewritten to accept these answer labels.
        select = lambda rows: sorted([{
            'node_id': row.get('node_id'),
            'kind': row.get('kind', row.get('type')),
            'identity': row.get('identity'),
            'hashes': row.get('hashes', row.get('digests')),
        } for row in rows], key=lambda row: row['node_id'])
        check('all_selected_module_identities', select(answer.get('modules', [])), select(reference['modules']))
    elif domain == 'native':
        for key in ['entry_address', 'caller_count', 'callee_count']:
            check(key, answer.get(key), reference[key])
        prefix = answer.get('pseudocode_prefix')
        if isinstance(prefix, dict) and prefix.get('unit') == 'utf16-code-units':
            text = prefix.get('text')
            if 'requested' in prefix:
                check('pseudocode_prefix.requested', prefix['requested'], 512)
            if 'returned' in prefix:
                check('pseudocode_prefix.returned', prefix['returned'], len(reference['pseudocode_prefix'].encode('utf-16-le')) // 2)
            prefix = text
        check('pseudocode_prefix', prefix, reference['pseudocode_prefix'])
        containing = answer.get('containing_procedure')
        expected_containing = reference['containing_procedure']['procedure']
        if isinstance(containing, str):
            # Compare the explicitly reported display to the known source;
            # never parse a display name into a canonical lookup identity.
            check('containing_procedure.display', containing,
                  expected_containing['name'] + ' @ ' + expected_containing['address'])
        else:
            containing = containing if isinstance(containing, dict) else {}
            nested = containing.get('procedure')
            nested = nested if isinstance(nested, dict) else {}
            for key in ['address', 'name']:
                check('containing_procedure.' + key, containing.get(key, nested.get(key)), expected_containing[key])
        procedure = answer.get('procedure')
        procedure = procedure if isinstance(procedure, dict) else {}
        expected = reference['procedure']
        for key in ['address', 'name', 'classification', 'signature', 'locals']:
            check('procedure.' + key, procedure.get(key), expected[key])
        body = procedure.get('body')
        body = body if isinstance(body, dict) else {}
        check('procedure.body_ranges', procedure.get('body_ranges', body.get('ranges', body.get('ranges_inclusive'))), expected['body']['ranges'])
        check('procedure.total_bytes', procedure.get('total_bytes', body.get('total_bytes')), expected['body']['total_bytes'])
        if 'load_image' in reference:
            sources = [source for source in producer_results.get('inspect_native_load_image', [])
                       if isinstance(source, dict)
                       and source.get('operation') == 'inspect_native_load_image'
                       and isinstance(source.get('evidence_id'), str)
                       and source.get('evidence_id') in reported_ids
                       and isinstance(source.get('normalized_result'), dict)]
            checks['load_image_source_observed_and_referenced'] = bool(sources)
            load = sources[-1]['normalized_result'] if sources else {}
            entry = load.get('entry')
            entry = entry if isinstance(entry, dict) else {}
            check('load_image.target_sha256', load.get('target_sha256'), reference['artifact_sha256'])
            check('load_image.source_status', load.get('status'), reference['load_image']['status'])
            check('load_image.source_entry', entry.get('linear_address'), reference['entry_address'])
            status = answer.get('load_image_status')
            check('load_image_status', status.get('status') if isinstance(status, dict) else status, load.get('status'))
            if isinstance(status, dict):
                for key in ['format', 'load_segment', 'header_bytes', 'module_bytes', 'overlay_bytes']:
                    if key in status:
                        check('load_image.' + key, status[key], load.get(key))
                if 'entry_linear_address' in status:
                    check('load_image.entry_linear_address', status['entry_linear_address'], entry.get('linear_address'))
        calls = summary['completed_mcp_calls']
        checks['default_analysis_not_disabled'] = all(not call['arguments'].get('skip_auto_analysis', False) for call in calls if call['tool'] == 'open_binary')
        checks['session_close_completed'] = any(call['tool'] == 'close_binary' and call['status'] == 'completed' for call in calls)
    elif domain == 'browser':
        source = producer_results['capture_browser_scenario'][-1]
        n = source['normalized_result']
        for key in ['browser', 'completeness']:
            check(key, answer.get(key), n[key])
        check('start_origin', answer.get('start_origin'), n['scenario']['start_origin'])
        step_count = answer.get('step_count')
        check('step_count', step_count.get('captured_steps') if isinstance(step_count, dict) else step_count, len(n['steps']))
        final = n['steps'][-1]
        expected_url = final['after_url'] if isinstance(answer.get('final_url'), dict) else final['after_url']['url']
        check('final_url', answer.get('final_url'), expected_url)
        for key in ['dom', 'accessibility']:
            expected = final['artifacts'][key]
            actual = answer.get(key, {})
            if expected['state'] == 'captured':
                actual = actual.get('value', actual)
                for field in ['sha256', 'bytes']:
                    check(key + '.' + field, actual.get(field), expected['value'][field])
            else:
                check(key + '.state', actual.get('state'), expected['state'])
        checks['owned_browser_terminated'] = n['browser']['cleanup'] == 'terminated-owned-process'
        events = n['events']['items']
        page_errors = [event for event in events if event['kind'] == 'page-error']
        if isinstance(answer.get('page_errors'), dict):
            check('page_error_count', answer['page_errors'].get('count'), len(page_errors))
        network = answer.get('network', {})
        if 'observed_transactions' in network:
            check('network_transaction_count', network['observed_transactions'], len({event['transaction_id'] for event in events if event.get('transaction_id') is not None}))
        if 'observed_response_statuses' in network:
            statuses = {}
            for event in events:
                if event['kind'] == 'response':
                    status = str(event['status'])
                    statuses[status] = statuses.get(status, 0) + 1
            check('network_response_statuses', network['observed_response_statuses'], statuses)
    factual_checks = dict(checks)
    checks.update(workflow_checks)
    scopes = [json.loads(line) for line in args.resource_log.read_text().splitlines() if line.startswith('{')]
    scope = next(value for value in reversed(scopes) if 'resource_scope' in value)
    checks['no_resource_abort'] = scope['abort'] is None
    events = scope.get('resource_events_final')
    checks['resource_counters_verified'] = scope.get('resource_counters_verified') is True
    checks['no_observed_oom_or_task_limit'] = (
        isinstance(events, dict)
        and all(events.get('memory.events', {}).get(k) == 0 for k in ['max', 'oom', 'oom_kill'])
        and events.get('pids.events', {}).get('max') == 0
    )
    checks['no_resource_limit_event'] = (
        checks['resource_counters_verified']
        and checks['no_observed_oom_or_task_limit']
        and events['memory.events'].get('high') == 0
    )
    report = {
        'review_source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
        'domain': domain, 'checks': checks, 'all_selected_checks_pass': all(checks.values()),
        'factual_checks': factual_checks, 'all_factual_checks_pass': all(factual_checks.values()),
        **({'workflow_checks': workflow_checks, 'all_workflow_checks_pass': all(workflow_checks.values())}
           if domain == 'pcap' else {}),
        'differences': differences, 'tool_errors': tool_errors, 'summary': summary,
        'fingerprints': {key: value for key, value in settings.items() if 'sha256' in key or key in ['runtime_revision', 'enabled_tools', 'authority']},
        'answer_file_sha256': hashlib.sha256((args.trial / 'answer.txt').read_bytes()).hexdigest(),
        'answer_had_json_fence': fenced, 'reported_evidence_ids': sorted(reported_ids), 'resource': scope,
        'scope': 'Fixed selected-fact review against actual public producer observations. Equivalent explicitly named answer representations are accepted; unknowns/nulls remain checked. Limitations, unsupported claims and broader behavior still require transcript review. Token usage and failed calls are retained; zero exit status is not a factual grade.',
    }
    with args.output.open('x') as output:
        output.write(json.dumps(report, indent=2, ensure_ascii=False) + '\n')
    print(json.dumps({'domain': domain, 'passed': report['all_selected_checks_pass'], 'failed_checks': [key for key, value in checks.items() if not value]}))
    raise SystemExit(0 if report['all_selected_checks_pass'] else 1)


if __name__ == '__main__':
    main()