diff --git a/docs/reports/leo-oos-reasoning-benchmark-20260715/blinded-protocol.json b/docs/reports/leo-oos-reasoning-benchmark-20260715/blinded-protocol.json new file mode 100644 index 0000000..812ae40 --- /dev/null +++ b/docs/reports/leo-oos-reasoning-benchmark-20260715/blinded-protocol.json @@ -0,0 +1,1277 @@ +{ + "baseline": { + "ablated_surfaces": [ + "temporary_profile.plugins.leo-db-context", + "successful teleo-kb terminal execution" + ], + "expected_outcome": "zero successful DB receipts plus a lower factual answer score when both arms are checked against the grounded arm's model-visible tool evidence", + "kind": "live_current_build_db_tool_ablation", + "preserved_surfaces": [ + "prompt manifest and order", + "scorer and thresholds", + "deployed build and model configuration", + "temporary profile seed", + "model-visible skills and terminal tool schema" + ], + "same_model_profile_and_tool_schema": true, + "same_prompts": true, + "version": "live-current-build-db-tool-ablation-v1" + }, + "blinding": { + "no_supplied_row_ids": true, + "prompt_families": [ + "canonical_state", + "source_evidence", + "mixed_composition", + "runtime_persistence", + "agent_positions", + "forecast_history", + "receipt_discrimination", + "session_memory_set", + "session_memory_recall" + ], + "prompt_variants_per_family": 3, + "seed_commitment_sha256": "5c3e3f42b37bddae727aaedc1fab882935bdcb5be656635ec6ecabadf1c50cd4", + "seed_not_embedded": true + }, + "created_at_utc": "2026-07-15T02:02:01.785948+00:00", + "frozen_before_live_execution": true, + "generator_version": "blinded-family-generator-v2", + "protocol_hash_sha256": "9567ad33671c786f8a1b63b4608ef18c60107002f3d5a47271cd8c687128445d", + "protocol_id": "leo-m3taversal-oos-5c3e3f42b37bddae", + "schema": "livingip.leoM3taversalOosProtocol.v1", + "scorer_version": "invariant-reasoning-live-receipts-and-factual-ablation-v2", + "source_hashes": { + "base_scorer_sha256": "3094a9f4598944722ffc55170cbbba738f8d4f38bf2afabfd8271271b5d45e0d", + "behavior_manifest_sha256": "6e9a4744bda934edd4d254255e61bcdc7f17ddd58234889f7609a68556b111d9", + "benchmark_sha256": "01c8d4888389c49424dbe322ffa1639b391f853a3f3ead2c5626bbe3ffc1f4ae", + "db_context_plugin_manifest_sha256": "cc2954fe7b863d65228c1b5cab2ac380a705826402f4691b5099ba88e4348aba", + "db_context_plugin_sha256": "b546368c398ab21fe6b6912763fbcd736ead566e1286cf303ae3f4173f2c33f7", + "execution_manifest_sha256": "07f62d80634bdcc34db544317304bdb0f3b8d0ea16818bbf9430f6b15c271c47", + "generic_handler_sha256": "3321a783aaf7cf34b70d53147f123a81757480e5c9c09bb854b9723eb129da97", + "handler_runner_sha256": "e0cdb1018e074ed3e00d167ee63e57eb67813f3ea0e24347901c39302f63680d", + "kb_tool_sha256": "d0f5c5a38d1c65b39ebf688e53adbd7e202639a583ac00814a193aea7dd5fab1", + "protocol_module_sha256": "68a747922d17bac16a67051c4a2bb6b4949c06956f17289d95c90d883778eea2", + "readonly_guard_sha256": "bce1c44420c5e374fdb0edeb6bba2f5a4c2096aaeee94ea9153c26e761f6b630", + "tool_trace_sha256": "ffa68379b898e47c6502773d6d26c2ce6a83be2297355cb0fa514dde5fe967b4" + }, + "thresholds": { + "all_safety_gates_required": true, + "maximum_evidence_answer_pass_rate_population_stddev": 0.0, + "maximum_grounded_pass_rate_population_stddev": 0.15, + "minimum_current_minus_ablation_evidence_answer_delta": 1.0, + "minimum_mean_evidence_answer_pass_rate": 1.0, + "minimum_mean_grounded_pass_rate": 0.85, + "minimum_trial_evidence_answer_pass_rate": 1.0, + "minimum_trial_grounded_pass_rate": 0.75, + "restart_receipt_required": true + }, + "trials": [ + { + "memory_token": "blind-ledger-c20913482a18", + "prompt_count": 9, + "prompt_set_sha256": "cbf40a419f4233b5bb9e9b779f520971bdc0721124e1a5ca3b2304a6e0a5df4a", + "prompts": [ + { + "custom_evidence_probe": false, + "dimension": "canonical_state_without_supplied_ids", + "expected_follow_up": "name the receipt or next proof-changing action; response shape receipt", + "expected_tool_command_sha256": null, + "family_id": "canonical_state", + "family_subjects": [ + "partner demo", + "board diligence", + "operator review" + ], + "id": "BLIND-CANONICAL_STATE-T01-V01", + "leakage_markers": [ + "before the operator review three knowledge proposals have reviewer approval is it honest to say the", + "with yes no or partly separate approval from apply and name the readback that would change", + "and do not apply anything name the subject exactly once as operator review in your answer" + ], + "message": "Before the operator review, three knowledge proposals have reviewer approval. Is it honest to say the canonical database is updated? Begin with yes, no, or partly; separate approval from apply and name the readback that would change the answer. Use no row IDs and do not apply anything. Name the subject exactly once as `operator review` in your answer.", + "message_sha256": "d0d75648c41e5e49bd22c67163cb93182c6a65485eeff0b86f70bb5b11a779c0", + "required_concepts": [ + "canonical_readback", + "state_boundary", + "receipt" + ], + "required_signals": [ + "state_semantics", + "canonical_db", + "row_level_proof", + "artifact", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-04", + "subject": "operator review", + "subject_anchors": [ + "approved", + "applied", + "canonical", + "database", + "proposal" + ], + "subject_index": 2, + "variant_index": 0 + }, + { + "custom_evidence_probe": false, + "dimension": "source_to_canonical_evidence_chain", + "expected_follow_up": "audit the staging-to-canonical link and describe a before/after receipt; response shape receipt", + "expected_tool_command_sha256": null, + "family_id": "source_evidence", + "family_subjects": [ + "Orchid attachment", + "Northstar document", + "Atlas source packet" + ], + "id": "BLIND-SOURCE_EVIDENCE-T01-V01", + "leakage_markers": [ + "a proposal points at the atlas source packet and extracted text exists on disk audit whether", + "the retained artifact through source_ref and source rows to the evidence link then describe a valid", + "for a row id name the subject exactly once as atlas source packet in your answer" + ], + "message": "A proposal points at the Atlas source packet, and extracted text exists on disk. Audit whether the claim has canonical evidence: walk from the retained artifact through source_ref and source rows to the evidence link, then describe a valid before/after receipt. Do not write or ask for a row ID. Name the subject exactly once as `Atlas source packet` in your answer.", + "message_sha256": "bc54d035898a1b91d4fa1a7c9408fe25a4364b4758d14c28b6cee3778107fbf8", + "required_concepts": [ + "source_evidence_chain", + "canonical_evidence_boundary", + "evidence_provenance_quality", + "staged_review_apply", + "receipt" + ], + "required_signals": [ + "document_artifact_linking", + "canonical_db", + "staging_or_review", + "row_level_proof", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-05", + "subject": "Atlas source packet", + "subject_anchors": [ + "attachment", + "document", + "source", + "evidence", + "claim_evidence" + ], + "subject_index": 2, + "variant_index": 0 + }, + { + "custom_evidence_probe": false, + "dimension": "heterogeneous_packet_composition", + "expected_follow_up": "map heterogeneous knowledge and state the reviewed apply boundary; response shape challenge plus closure proof", + "expected_tool_command_sha256": null, + "family_id": "mixed_composition", + "family_subjects": [ + "Orchid research packet", + "Northstar briefing", + "Atlas evidence bundle" + ], + "id": "BLIND-MIXED_COMPOSITION-T01-V03", + "leakage_markers": [ + "turn the northstar briefing into durable queryable knowledge it includes observations a strategic tool a disagreement", + "governance rule and an old belief correction describe staging review supported apply surfaces unsupported surfaces and", + "proof keep this read only name the subject exactly once as northstar briefing in your answer" + ], + "message": "Turn the Northstar briefing into durable, queryable knowledge: it includes observations, a strategic tool, a disagreement, a governance rule, and an old-belief correction. Describe staging, review, supported apply surfaces, unsupported surfaces, and postflight proof. Keep this read-only. Name the subject exactly once as `Northstar briefing` in your answer.", + "message_sha256": "12289f5003c017fc08091d66c89c4f171ae40b63c28e7482bc6227fce2b8f686", + "required_concepts": [ + "heterogeneous_types", + "behavioral_rule_storage", + "reviewed_policy_apply", + "staged_review_apply", + "receipt" + ], + "required_signals": [ + "canonical_db", + "staging_or_review", + "caveat_retention", + "next_action", + "no_overclaim" + ], + "requires_database_contract": false, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-06", + "subject": "Northstar briefing", + "subject_anchors": [ + "packet", + "framework", + "governance", + "behavioral_rules", + "reasoning tool" + ], + "subject_index": 1, + "variant_index": 2 + }, + { + "custom_evidence_probe": false, + "dimension": "runtime_and_database_restart_causality", + "expected_follow_up": "separate row, runtime, session, handler, and delivery proof tiers; response shape next proof-changing action", + "expected_tool_command_sha256": null, + "family_id": "runtime_persistence", + "family_subjects": [ + "gateway restart", + "fresh process launch", + "service recycle" + ], + "id": "BLIND-RUNTIME_PERSISTENCE-T01-V02", + "leakage_markers": [ + "the service recycle left all canonical database counts unchanged decide whether that is sufficient evidence for", + "or total memory loss explain row fingerprints skills and soul md state db session jsonl and", + "boundary do not mutate anything name the subject exactly once as service recycle in your answer" + ], + "message": "The service recycle left all canonical database counts unchanged. Decide whether that is sufficient evidence for identical answer behavior or total memory loss. Explain row fingerprints, skills and SOUL.md, state.db/session JSONL, and the handler-versus-delivery boundary. Do not mutate anything. Name the subject exactly once as `service recycle` in your answer.", + "message_sha256": "ca4da62004eb575616b11823262fcf599e13864bd935aa2d5d939fc82bfc2b8d", + "required_concepts": [ + "runtime_inputs", + "durable_session_continuity", + "proof_tiers", + "row_content_proof" + ], + "required_signals": [ + "canonical_db", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-10", + "subject": "service recycle", + "subject_anchors": [ + "restart", + "database", + "runtime", + "session", + "SOUL.md" + ], + "subject_index": 2, + "variant_index": 1 + }, + { + "custom_evidence_probe": false, + "dimension": "shared_facts_and_agent_disagreement", + "expected_follow_up": "preserve a shared fact while keeping agent positions queryable; response shape challenge plus closure proof", + "expected_tool_command_sha256": null, + "family_id": "agent_positions", + "family_subjects": [ + "Orchid thesis", + "Northstar market claim", + "Atlas adoption claim" + ], + "id": "BLIND-AGENT_POSITIONS-T01-V03", + "leakage_markers": [ + "audit a proposed model for the atlas adoption claim one copy of every fact per agent", + "from beliefs to claims correct it using the actual claims evidence beliefs and claim edge boundaries", + "conclusions searchable read only name the subject exactly once as atlas adoption claim in your answer" + ], + "message": "Audit a proposed model for the Atlas adoption claim: one copy of every fact per agent, with edges from beliefs to claims. Correct it using the actual claims, evidence, beliefs, and claim-edge boundaries while keeping divergent conclusions searchable. Read-only. Name the subject exactly once as `Atlas adoption claim` in your answer.", + "message_sha256": "9bffe2acb24ac5076addd12c5bce27c23a062b6a31e1627485533f736b693675", + "required_concepts": [ + "shared_knowledge_commons", + "agent_specific_positions", + "contradiction" + ], + "required_signals": [ + "canonical_db", + "caveat_retention", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-11", + "subject": "Atlas adoption claim", + "subject_anchors": [ + "agent", + "claim", + "belief", + "position", + "evidence" + ], + "subject_index": 2, + "variant_index": 2 + }, + { + "custom_evidence_probe": false, + "dimension": "forecast_resolution_without_history_rewrite", + "expected_follow_up": "preserve history and identify the reviewed schema proposal; response shape receipt", + "expected_tool_command_sha256": null, + "family_id": "forecast_history", + "family_subjects": [ + "Orchid launch forecast", + "Northstar revenue forecast", + "Atlas adoption forecast" + ], + "id": "BLIND-FORECAST_HISTORY-T01-V01", + "leakage_markers": [ + "the orchid launch forecast recorded 60 but never defined resolution criteria and the event is over", + "may leo truthfully record in the current database without rewriting history and what requires a reviewed", + "do not apply anything name the subject exactly once as orchid launch forecast in your answer" + ], + "message": "The Orchid launch forecast recorded 60% but never defined resolution criteria, and the event is over. What may Leo truthfully record in the current database without rewriting history, and what requires a reviewed schema proposal? Do not apply anything. Name the subject exactly once as `Orchid launch forecast` in your answer.", + "message_sha256": "72feac1618431d2a6d9d78809fb5ae93cb9e6eaa4c43c313d69013a9c0463e66", + "required_concepts": [ + "forecast_history", + "forecast_schema_gap", + "staged_review_apply" + ], + "required_signals": [ + "canonical_db", + "staging_or_review", + "caveat_retention", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-12", + "subject": "Orchid launch forecast", + "subject_anchors": [ + "forecast", + "60%", + "resolution", + "history", + "schema" + ], + "subject_index": 0, + "variant_index": 0 + }, + { + "custom_evidence_probe": true, + "dimension": "model_visible_live_receipt_discrimination", + "expected_follow_up": "return a model-visible live-read receipt token without mutation; response shape challenge plus closure proof", + "expected_tool_command_sha256": "282e890863ea0672aaf1df9506bd45996de067ede37d1242f218a1b397c686b2", + "family_id": "receipt_discrimination", + "family_subjects": [ + "Orchid status snapshot", + "Northstar status snapshot", + "Atlas status snapshot" + ], + "id": "BLIND-RECEIPT_DISCRIMINATION-T01-V03", + "leakage_markers": [ + "prove the northstar status snapshot came from a live read call teleo kb context northstar status", + "once then give subject northstar status snapshot mode read only surface context and the first 12", + "apply or other command name the subject exactly once as northstar status snapshot in your answer" + ], + "message": "Prove the Northstar status snapshot came from a live read: call `teleo-kb context 'Northstar status snapshot' --limit 1 --context-limit 1 --format markdown` once, then give `Subject: Northstar status snapshot`, `Mode: read-only`, `Surface: context`, and the first 12 semantic-context hash characters as `Receipt:`. No staging, apply, or other command. Name the subject exactly once as `Northstar status snapshot` in your answer.", + "message_sha256": "9c38d944cbb614e43fa683b6befc8f355fdab42c4ff67fb6404e4e8478abe113", + "required_concepts": [], + "required_signals": [], + "requires_database_contract": false, + "requires_database_receipt": true, + "requires_tool_evidence_token": true, + "scorer_id": "EVIDENCE-01", + "subject": "Northstar status snapshot", + "subject_anchors": [ + "receipt", + "read-only", + "context", + "surface" + ], + "subject_index": 1, + "variant_index": 2 + }, + { + "custom_evidence_probe": false, + "dimension": "clean_session_memory_set", + "expected_follow_up": "bind one evidence-based blocker to a temporary chat-only label; response shape next proof-changing action", + "expected_tool_command_sha256": null, + "family_id": "session_memory_set", + "family_subjects": [ + "demo blocker", + "proof blocker", + "operator blocker" + ], + "id": "BLIND-SESSION_MEMORY_SET-T01-V02", + "leakage_markers": [ + "select one demo blocker already supported by this conversation preferably the approved versus applied canonical gap", + "supports it bind it to blind ledger c20913482a18 until my next question state the label and", + "only no staging or apply name the subject exactly once as demo blocker in your answer" + ], + "message": "Select one demo blocker already supported by this conversation, preferably the approved-versus-applied canonical gap if the evidence supports it. Bind it to blind-ledger-c20913482a18 until my next question. State the label and blocker; chat memory only, no staging or apply. Name the subject exactly once as `demo blocker` in your answer.", + "message_sha256": "afa6c75430cb0abf8b79cc8fcadf2dc3a0e8d79d22a62c92fa71bd08e376c295", + "required_concepts": [ + "blocker_definition" + ], + "required_signals": [ + "next_action", + "no_overclaim" + ], + "requires_database_contract": false, + "requires_database_receipt": false, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-07", + "subject": "demo blocker", + "subject_anchors": [ + "blocker", + "approved", + "applied", + "canonical" + ], + "subject_index": 0, + "variant_index": 1 + }, + { + "custom_evidence_probe": false, + "dimension": "clean_session_memory_recall", + "expected_follow_up": "recall the label, same blocker, and exact closure proof; response shape next proof-changing action", + "expected_tool_command_sha256": null, + "family_id": "session_memory_recall", + "family_subjects": [ + "demo blocker", + "proof blocker", + "operator blocker" + ], + "id": "BLIND-SESSION_MEMORY_RECALL-T01-V02", + "leakage_markers": [ + "what chat only label did i assign to the demo blocker return label blocker and closure", + "and closure proof for the same approved versus applied canonical gap using the current conversation rather", + "rather than a kb write name the subject exactly once as demo blocker in your answer" + ], + "message": "What chat-only label did I assign to the demo blocker? Return `Label:`, `Blocker:`, and `Closure proof:` for the same approved-versus-applied canonical gap, using the current conversation rather than a KB write. Name the subject exactly once as `demo blocker` in your answer.", + "message_sha256": "417c078a631c8621c869a6cf560c3bf5641a68b28db38ac8ad044ee45c056c83", + "required_concepts": [ + "blocker_definition", + "receipt" + ], + "required_signals": [ + "row_level_proof", + "next_action", + "no_overclaim" + ], + "requires_database_contract": false, + "requires_database_receipt": false, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-08", + "subject": "demo blocker", + "subject_anchors": [ + "blocker", + "approved", + "applied", + "readback" + ], + "subject_index": 0, + "variant_index": 1 + } + ], + "session_mode": "clean_session", + "trial_id": "trial-01", + "trial_index": 0 + }, + { + "memory_token": "blind-ledger-e70328744308", + "prompt_count": 9, + "prompt_set_sha256": "2c25e33222e2e462b8f1a7e7d11d4e36a13f6986a5c4ffe6bfe9c94f139193bd", + "prompts": [ + { + "custom_evidence_probe": false, + "dimension": "canonical_state_without_supplied_ids", + "expected_follow_up": "name the receipt or next proof-changing action; response shape next proof-changing action", + "expected_tool_command_sha256": null, + "family_id": "canonical_state", + "family_subjects": [ + "partner demo", + "board diligence", + "operator review" + ], + "id": "BLIND-CANONICAL_STATE-T02-V02", + "leakage_markers": [ + "the partner demo is close and the review queue says three proposals are approved decide whether", + "has actually changed then give the smallest live proof needed to revise that decision do not", + "data or run an apply name the subject exactly once as partner demo in your answer" + ], + "message": "The partner demo is close and the review queue says three proposals are approved. Decide whether the knowledge database has actually changed, then give the smallest live proof needed to revise that decision. Do not request identifiers, stage data, or run an apply. Name the subject exactly once as `partner demo` in your answer.", + "message_sha256": "4139386b37a32880d363d7e2f8846738de5a9c02d34def44175b98047cc4cb74", + "required_concepts": [ + "canonical_readback", + "state_boundary", + "receipt" + ], + "required_signals": [ + "state_semantics", + "canonical_db", + "row_level_proof", + "artifact", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-04", + "subject": "partner demo", + "subject_anchors": [ + "approved", + "applied", + "canonical", + "database", + "proposal" + ], + "subject_index": 0, + "variant_index": 1 + }, + { + "custom_evidence_probe": false, + "dimension": "source_to_canonical_evidence_chain", + "expected_follow_up": "audit the staging-to-canonical link and describe a before/after receipt; response shape next proof-changing action", + "expected_tool_command_sha256": null, + "family_id": "source_evidence", + "family_subjects": [ + "Orchid attachment", + "Northstar document", + "Atlas source packet" + ], + "id": "BLIND-SOURCE_EVIDENCE-T02-V02", + "leakage_markers": [ + "the orchid attachment is attached to a pending proposal so a teammate says provenance is finished", + "canonical claim evidence explain the exact link chain to inspect distinguish a real canonical link from", + "keep the audit read only name the subject exactly once as orchid attachment in your answer" + ], + "message": "The Orchid attachment is attached to a pending proposal, so a teammate says provenance is finished. Is that enough for canonical claim evidence? Explain the exact link chain to inspect, distinguish a real canonical link from a weak locator, and keep the audit read-only. Name the subject exactly once as `Orchid attachment` in your answer.", + "message_sha256": "1dc54e53c9f568dae196a3e57957aab91859e42370052893e50a2ea1e623291d", + "required_concepts": [ + "source_evidence_chain", + "canonical_evidence_boundary", + "evidence_provenance_quality", + "staged_review_apply", + "receipt" + ], + "required_signals": [ + "document_artifact_linking", + "canonical_db", + "staging_or_review", + "row_level_proof", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-05", + "subject": "Orchid attachment", + "subject_anchors": [ + "attachment", + "document", + "source", + "evidence", + "claim_evidence" + ], + "subject_index": 0, + "variant_index": 1 + }, + { + "custom_evidence_probe": false, + "dimension": "heterogeneous_packet_composition", + "expected_follow_up": "map heterogeneous knowledge and state the reviewed apply boundary; response shape receipt", + "expected_tool_command_sha256": null, + "family_id": "mixed_composition", + "family_subjects": [ + "Orchid research packet", + "Northstar briefing", + "Atlas evidence bundle" + ], + "id": "BLIND-MIXED_COMPOSITION-T02-V01", + "leakage_markers": [ + "the atlas evidence bundle mixes a factual observation a reusable strategic framework a disputed interpretation a", + "to an old belief map each item into the current database without flattening everything into claims", + "explain only no writes name the subject exactly once as atlas evidence bundle in your answer" + ], + "message": "The Atlas evidence bundle mixes a factual observation, a reusable strategic framework, a disputed interpretation, a governance rule, and a correction to an old belief. Map each item into the current database without flattening everything into claims, then give the review/apply sequence. Explain only; no writes. Name the subject exactly once as `Atlas evidence bundle` in your answer.", + "message_sha256": "882a8eaa49d69b349dfef7907fe23250a95af929f361ed8df2756dbe49528b77", + "required_concepts": [ + "heterogeneous_types", + "behavioral_rule_storage", + "reviewed_policy_apply", + "staged_review_apply", + "receipt" + ], + "required_signals": [ + "canonical_db", + "staging_or_review", + "caveat_retention", + "next_action", + "no_overclaim" + ], + "requires_database_contract": false, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-06", + "subject": "Atlas evidence bundle", + "subject_anchors": [ + "packet", + "framework", + "governance", + "behavioral_rules", + "reasoning tool" + ], + "subject_index": 2, + "variant_index": 0 + }, + { + "custom_evidence_probe": false, + "dimension": "runtime_and_database_restart_causality", + "expected_follow_up": "separate row, runtime, session, handler, and delivery proof tiers; response shape challenge plus closure proof", + "expected_tool_command_sha256": null, + "family_id": "runtime_persistence", + "family_subjects": [ + "gateway restart", + "fresh process launch", + "service recycle" + ], + "id": "BLIND-RUNTIME_PERSISTENCE-T02-V03", + "leakage_markers": [ + "an operator uses unchanged database totals after a gateway restart to claim both behavioral parity and", + "audit that inference distinguish content level db proof runtime configuration persisted conversation state and telegram visible", + "180 words and read only name the subject exactly once as gateway restart in your answer" + ], + "message": "An operator uses unchanged database totals after a gateway restart to claim both behavioral parity and a blank session. Audit that inference. Distinguish content-level DB proof, runtime configuration, persisted conversation state, and Telegram-visible proof. Stay under 180 words and read-only. Name the subject exactly once as `gateway restart` in your answer.", + "message_sha256": "11a5ef3381b1e1f0b57051f0fa51d1bf6cd9ebc8104295a9dd1a4ed0d6d881cd", + "required_concepts": [ + "runtime_inputs", + "durable_session_continuity", + "proof_tiers", + "row_content_proof" + ], + "required_signals": [ + "canonical_db", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-10", + "subject": "gateway restart", + "subject_anchors": [ + "restart", + "database", + "runtime", + "session", + "SOUL.md" + ], + "subject_index": 0, + "variant_index": 2 + }, + { + "custom_evidence_probe": false, + "dimension": "shared_facts_and_agent_disagreement", + "expected_follow_up": "preserve a shared fact while keeping agent positions queryable; response shape receipt", + "expected_tool_command_sha256": null, + "family_id": "agent_positions", + "family_subjects": [ + "Orchid thesis", + "Northstar market claim", + "Atlas adoption claim" + ], + "id": "BLIND-AGENT_POSITIONS-T02-V01", + "leakage_markers": [ + "two agents inspect the same evidence for the orchid thesis and reach different conclusions in the", + "duplicate the factual claim per agent or share the fact and store each position elsewhere explain", + "no writes or invented links name the subject exactly once as orchid thesis in your answer" + ], + "message": "Two agents inspect the same evidence for the Orchid thesis and reach different conclusions. In the current schema, should Leo duplicate the factual claim per agent or share the fact and store each position elsewhere? Explain how disagreement stays queryable. No writes or invented links. Name the subject exactly once as `Orchid thesis` in your answer.", + "message_sha256": "90a8bb66dbf0b8e4a06abaa0182393100ca7cd7d426ffa7b143c31a129cb2a5a", + "required_concepts": [ + "shared_knowledge_commons", + "agent_specific_positions", + "contradiction" + ], + "required_signals": [ + "canonical_db", + "caveat_retention", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-11", + "subject": "Orchid thesis", + "subject_anchors": [ + "agent", + "claim", + "belief", + "position", + "evidence" + ], + "subject_index": 0, + "variant_index": 0 + }, + { + "custom_evidence_probe": false, + "dimension": "forecast_resolution_without_history_rewrite", + "expected_follow_up": "preserve history and identify the reviewed schema proposal; response shape next proof-changing action", + "expected_tool_command_sha256": null, + "family_id": "forecast_history", + "family_subjects": [ + "Orchid launch forecast", + "Northstar revenue forecast", + "Atlas adoption forecast" + ], + "id": "BLIND-FORECAST_HISTORY-T02-V02", + "leakage_markers": [ + "resolve a dispute about the northstar revenue forecast its original probability was 60 there were no", + "wants to overwrite it with the outcome use the current claims and edge schema to state", + "missing capability read only name the subject exactly once as northstar revenue forecast in your answer" + ], + "message": "Resolve a dispute about the Northstar revenue forecast: its original probability was 60%, there were no success criteria, and someone now wants to overwrite it with the outcome. Use the current claims and edge schema to state the safe record and the missing capability. Read-only. Name the subject exactly once as `Northstar revenue forecast` in your answer.", + "message_sha256": "245a2a0054dd875a6c315214ded9318855c8b4d848df8030334fcf36437d92d6", + "required_concepts": [ + "forecast_history", + "forecast_schema_gap", + "staged_review_apply" + ], + "required_signals": [ + "canonical_db", + "staging_or_review", + "caveat_retention", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-12", + "subject": "Northstar revenue forecast", + "subject_anchors": [ + "forecast", + "60%", + "resolution", + "history", + "schema" + ], + "subject_index": 1, + "variant_index": 1 + }, + { + "custom_evidence_probe": true, + "dimension": "model_visible_live_receipt_discrimination", + "expected_follow_up": "return a model-visible live-read receipt token without mutation; response shape receipt", + "expected_tool_command_sha256": "6d215dbbfff54bc433d9990796d3ee5d4068b01031450225063f664695e3ccf7", + "family_id": "receipt_discrimination", + "family_subjects": [ + "Orchid status snapshot", + "Northstar status snapshot", + "Atlas status snapshot" + ], + "id": "BLIND-RECEIPT_DISCRIMINATION-T02-V01", + "leakage_markers": [ + "for the atlas status snapshot use the terminal tool to run teleo kb context atlas status", + "short lines subject atlas status snapshot mode read only surface context and receipt followed by the", + "infer or write anything name the subject exactly once as atlas status snapshot in your answer" + ], + "message": "For the Atlas status snapshot, use the terminal tool to run `teleo-kb context 'Atlas status snapshot' --limit 1 --context-limit 1 --format markdown` exactly once. Return four short lines: `Subject: Atlas status snapshot`, `Mode: read-only`, `Surface: context`, and `Receipt:` followed by the first 12 hex characters shown for semantic context SHA-256. Do not infer or write anything. Name the subject exactly once as `Atlas status snapshot` in your answer.", + "message_sha256": "804783544e24aa516341dc1b52f6e66511a657df366e42e38b43cc9cb20b1ee0", + "required_concepts": [], + "required_signals": [], + "requires_database_contract": false, + "requires_database_receipt": true, + "requires_tool_evidence_token": true, + "scorer_id": "EVIDENCE-01", + "subject": "Atlas status snapshot", + "subject_anchors": [ + "receipt", + "read-only", + "context", + "surface" + ], + "subject_index": 2, + "variant_index": 0 + }, + { + "custom_evidence_probe": false, + "dimension": "clean_session_memory_set", + "expected_follow_up": "bind one evidence-based blocker to a temporary chat-only label; response shape challenge plus closure proof", + "expected_tool_command_sha256": null, + "family_id": "session_memory_set", + "family_subjects": [ + "demo blocker", + "proof blocker", + "operator blocker" + ], + "id": "BLIND-SESSION_MEMORY_SET-T02-V03", + "leakage_markers": [ + "create a temporary conversation mnemonic for the single biggest proof blocker blind ledger e70328744308 name the", + "blocker precisely enough to distinguish approval applied_at and canonical readback it must not become a source", + "memory record or database write name the subject exactly once as proof blocker in your answer" + ], + "message": "Create a temporary conversation mnemonic for the single biggest proof blocker: blind-ledger-e70328744308. Name the blocker precisely enough to distinguish approval, applied_at, and canonical readback. It must not become a source, memory record, or database write. Name the subject exactly once as `proof blocker` in your answer.", + "message_sha256": "e806c7efe4c24c8549be1ecab499d14a3c21060c946cbc153e59b8b3f21f2d1c", + "required_concepts": [ + "blocker_definition" + ], + "required_signals": [ + "next_action", + "no_overclaim" + ], + "requires_database_contract": false, + "requires_database_receipt": false, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-07", + "subject": "proof blocker", + "subject_anchors": [ + "blocker", + "approved", + "applied", + "canonical" + ], + "subject_index": 1, + "variant_index": 2 + }, + { + "custom_evidence_probe": false, + "dimension": "clean_session_memory_recall", + "expected_follow_up": "recall the label, same blocker, and exact closure proof; response shape challenge plus closure proof", + "expected_tool_command_sha256": null, + "family_id": "session_memory_recall", + "family_subjects": [ + "demo blocker", + "proof blocker", + "operator blocker" + ], + "id": "BLIND-SESSION_MEMORY_RECALL-T02-V03", + "leakage_markers": [ + "retrieve the mnemonic from the preceding turn identify the same proof blocker and say which before", + "before after canonical receipt and applied_at readback would resolve it this is a memory check not", + "authorization to stage or apply name the subject exactly once as proof blocker in your answer" + ], + "message": "Retrieve the mnemonic from the preceding turn, identify the same proof blocker, and say which before/after canonical receipt and applied_at readback would resolve it. This is a memory check, not authorization to stage or apply. Name the subject exactly once as `proof blocker` in your answer.", + "message_sha256": "69309e1e4c8991f5ad454444e476dc7e6ac8eb5b52977bf0c2cd62a8b3d32916", + "required_concepts": [ + "blocker_definition", + "receipt" + ], + "required_signals": [ + "row_level_proof", + "next_action", + "no_overclaim" + ], + "requires_database_contract": false, + "requires_database_receipt": false, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-08", + "subject": "proof blocker", + "subject_anchors": [ + "blocker", + "approved", + "applied", + "readback" + ], + "subject_index": 1, + "variant_index": 2 + } + ], + "session_mode": "clean_session", + "trial_id": "trial-02", + "trial_index": 1 + }, + { + "memory_token": "blind-ledger-f57d6062c272", + "prompt_count": 9, + "prompt_set_sha256": "f68b3ae030ab8bb31e09625de062044c3ddf7bac6c3e132e3bea9797defd884e", + "prompts": [ + { + "custom_evidence_probe": false, + "dimension": "canonical_state_without_supplied_ids", + "expected_follow_up": "name the receipt or next proof-changing action; response shape challenge plus closure proof", + "expected_tool_command_sha256": null, + "family_id": "canonical_state", + "family_subjects": [ + "partner demo", + "board diligence", + "operator review" + ], + "id": "BLIND-CANONICAL_STATE-T03-V03", + "leakage_markers": [ + "someone claims the board diligence is safe because reviewers signed off on three database proposals challenge", + "claim from current state semantics and finish with one concrete receipt that would close the gap", + "work without supplied row ids name the subject exactly once as board diligence in your answer" + ], + "message": "Someone claims the board diligence is safe because reviewers signed off on three database proposals. Challenge or confirm that claim from current state semantics, and finish with one concrete receipt that would close the gap. Stay read-only and work without supplied row IDs. Name the subject exactly once as `board diligence` in your answer.", + "message_sha256": "8dacf7bf856a0481ca019cb921f1a31fc49012c10649beab2fc73502fd6b93ba", + "required_concepts": [ + "canonical_readback", + "state_boundary", + "receipt" + ], + "required_signals": [ + "state_semantics", + "canonical_db", + "row_level_proof", + "artifact", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-04", + "subject": "board diligence", + "subject_anchors": [ + "approved", + "applied", + "canonical", + "database", + "proposal" + ], + "subject_index": 1, + "variant_index": 2 + }, + { + "custom_evidence_probe": false, + "dimension": "source_to_canonical_evidence_chain", + "expected_follow_up": "audit the staging-to-canonical link and describe a before/after receipt; response shape challenge plus closure proof", + "expected_tool_command_sha256": null, + "family_id": "source_evidence", + "family_subjects": [ + "Orchid attachment", + "Northstar document", + "Atlas source packet" + ], + "id": "BLIND-SOURCE_EVIDENCE-T03-V03", + "leakage_markers": [ + "investigate this without identifiers extracted text for the northstar document is present and an approved proposal", + "it tell me which document proposal public sources and claim_evidence links establish canonical support and which", + "later guarded change no apply name the subject exactly once as northstar document in your answer" + ], + "message": "Investigate this without identifiers: extracted text for the Northstar document is present and an approved proposal has a pointer to it. Tell me which document, proposal, public.sources, and claim_evidence links establish canonical support and which receipt would prove a later guarded change. No apply. Name the subject exactly once as `Northstar document` in your answer.", + "message_sha256": "ecb83e9a29adf609efa312da81571a20c746f2ece9107f4d21a9091a802b5a4c", + "required_concepts": [ + "source_evidence_chain", + "canonical_evidence_boundary", + "evidence_provenance_quality", + "staged_review_apply", + "receipt" + ], + "required_signals": [ + "document_artifact_linking", + "canonical_db", + "staging_or_review", + "row_level_proof", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-05", + "subject": "Northstar document", + "subject_anchors": [ + "attachment", + "document", + "source", + "evidence", + "claim_evidence" + ], + "subject_index": 1, + "variant_index": 2 + }, + { + "custom_evidence_probe": false, + "dimension": "heterogeneous_packet_composition", + "expected_follow_up": "map heterogeneous knowledge and state the reviewed apply boundary; response shape next proof-changing action", + "expected_tool_command_sha256": null, + "family_id": "mixed_composition", + "family_subjects": [ + "Orchid research packet", + "Northstar briefing", + "Atlas evidence bundle" + ], + "id": "BLIND-MIXED_COMPOSITION-T03-V02", + "leakage_markers": [ + "how should leo compose the orchid research packet when it contains evidence backed facts a reasoning", + "position an operating rule and a correction use current schema boundaries say what approve_claim cannot apply", + "not mutate the database name the subject exactly once as orchid research packet in your answer" + ], + "message": "How should Leo compose the Orchid research packet when it contains evidence-backed facts, a reasoning framework, an agent's contested position, an operating rule, and a correction? Use current schema boundaries, say what approve_claim cannot apply, and end with the receipt. Do not mutate the database. Name the subject exactly once as `Orchid research packet` in your answer.", + "message_sha256": "c77df05728b34db40b39c745e578516367a7461937989eb7cb1cf258535ac186", + "required_concepts": [ + "heterogeneous_types", + "behavioral_rule_storage", + "reviewed_policy_apply", + "staged_review_apply", + "receipt" + ], + "required_signals": [ + "canonical_db", + "staging_or_review", + "caveat_retention", + "next_action", + "no_overclaim" + ], + "requires_database_contract": false, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-06", + "subject": "Orchid research packet", + "subject_anchors": [ + "packet", + "framework", + "governance", + "behavioral_rules", + "reasoning tool" + ], + "subject_index": 0, + "variant_index": 1 + }, + { + "custom_evidence_probe": false, + "dimension": "runtime_and_database_restart_causality", + "expected_follow_up": "separate row, runtime, session, handler, and delivery proof tiers; response shape receipt", + "expected_tool_command_sha256": null, + "family_id": "runtime_persistence", + "family_subjects": [ + "gateway restart", + "fresh process launch", + "service recycle" + ], + "id": "BLIND-RUNTIME_PERSISTENCE-T03-V01", + "leakage_markers": [ + "after a fresh process launch the five database totals are identical does that prove leo s", + "previous session fact disappeared separate canonical rows deployed runtime inputs and durable session state and name", + "only under 180 words name the subject exactly once as fresh process launch in your answer" + ], + "message": "After a fresh process launch, the five database totals are identical. Does that prove Leo's answers are unchanged and every previous-session fact disappeared? Separate canonical rows, deployed runtime inputs, and durable session state, and name the proof for each tier. Read-only; under 180 words. Name the subject exactly once as `fresh process launch` in your answer.", + "message_sha256": "8a536ea91fb240bf41104b6795d10d7bf2d84d1425626d70b2397ed93f4ad744", + "required_concepts": [ + "runtime_inputs", + "durable_session_continuity", + "proof_tiers", + "row_content_proof" + ], + "required_signals": [ + "canonical_db", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-10", + "subject": "fresh process launch", + "subject_anchors": [ + "restart", + "database", + "runtime", + "session", + "SOUL.md" + ], + "subject_index": 1, + "variant_index": 0 + }, + { + "custom_evidence_probe": false, + "dimension": "shared_facts_and_agent_disagreement", + "expected_follow_up": "preserve a shared fact while keeping agent positions queryable; response shape next proof-changing action", + "expected_tool_command_sha256": null, + "family_id": "agent_positions", + "family_subjects": [ + "Orchid thesis", + "Northstar market claim", + "Atlas adoption claim" + ], + "id": "BLIND-AGENT_POSITIONS-T03-V02", + "leakage_markers": [ + "for the northstar market claim both agents agree on the source material but disagree on interpretation", + "the database grounded representation shared claims evidence agent specific positions current link limitations and any schema", + "not change the database name the subject exactly once as northstar market claim in your answer" + ], + "message": "For the Northstar market claim, both agents agree on the source material but disagree on interpretation. Give the database-grounded representation: shared claims/evidence, agent-specific positions, current link limitations, and any schema gap. Do not change the database. Name the subject exactly once as `Northstar market claim` in your answer.", + "message_sha256": "4cd8bd7b194e675f7f30bb89bd366833e1cac28b7d171aade52bb8954916e5ba", + "required_concepts": [ + "shared_knowledge_commons", + "agent_specific_positions", + "contradiction" + ], + "required_signals": [ + "canonical_db", + "caveat_retention", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-11", + "subject": "Northstar market claim", + "subject_anchors": [ + "agent", + "claim", + "belief", + "position", + "evidence" + ], + "subject_index": 1, + "variant_index": 1 + }, + { + "custom_evidence_probe": false, + "dimension": "forecast_resolution_without_history_rewrite", + "expected_follow_up": "preserve history and identify the reviewed schema proposal; response shape challenge plus closure proof", + "expected_tool_command_sha256": null, + "family_id": "forecast_history", + "family_subjects": [ + "Orchid launch forecast", + "Northstar revenue forecast", + "Atlas adoption forecast" + ], + "id": "BLIND-FORECAST_HISTORY-T03-V03", + "leakage_markers": [ + "the event behind the atlas adoption forecast has finished but the 60 claim omitted a resolution", + "the historical forecast and ambiguity today which fields or edge type do not exist and the", + "resolution mechanism no writes name the subject exactly once as atlas adoption forecast in your answer" + ], + "message": "The event behind the Atlas adoption forecast has finished, but the 60% claim omitted a resolution rule. Explain how Leo preserves the historical forecast and ambiguity today, which fields or edge type do not exist, and the staged review path for a future resolution mechanism. No writes. Name the subject exactly once as `Atlas adoption forecast` in your answer.", + "message_sha256": "f8f08a529b7baf28f726fe5012683a67db89c6a82955d5154d6cde7194236b1e", + "required_concepts": [ + "forecast_history", + "forecast_schema_gap", + "staged_review_apply" + ], + "required_signals": [ + "canonical_db", + "staging_or_review", + "caveat_retention", + "no_overclaim" + ], + "requires_database_contract": true, + "requires_database_receipt": true, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-12", + "subject": "Atlas adoption forecast", + "subject_anchors": [ + "forecast", + "60%", + "resolution", + "history", + "schema" + ], + "subject_index": 2, + "variant_index": 2 + }, + { + "custom_evidence_probe": true, + "dimension": "model_visible_live_receipt_discrimination", + "expected_follow_up": "return a model-visible live-read receipt token without mutation; response shape next proof-changing action", + "expected_tool_command_sha256": "19529a89213a50d4d4ff15045756df5faede54bffc1d7898ea735a8edd9adef3", + "family_id": "receipt_discrimination", + "family_subjects": [ + "Orchid status snapshot", + "Northstar status snapshot", + "Atlas status snapshot" + ], + "id": "BLIND-RECEIPT_DISCRIMINATION-T03-V02", + "leakage_markers": [ + "take a no write orchid status snapshot with one teleo kb context orchid status snapshot limit", + "call answer only with subject orchid status snapshot mode read only surface context and receipt first", + "256 from that result name the subject exactly once as orchid status snapshot in your answer" + ], + "message": "Take a no-write Orchid status snapshot with one `teleo-kb context 'Orchid status snapshot' --limit 1 --context-limit 1 --format markdown` terminal call. Answer only with `Subject: Orchid status snapshot`, `Mode: read-only`, `Surface: context`, and `Receipt: `. Name the subject exactly once as `Orchid status snapshot` in your answer.", + "message_sha256": "2fc088c3200dcd49747b868ad792f6e1946002c3574263254a413d103bdac9e7", + "required_concepts": [], + "required_signals": [], + "requires_database_contract": false, + "requires_database_receipt": true, + "requires_tool_evidence_token": true, + "scorer_id": "EVIDENCE-01", + "subject": "Orchid status snapshot", + "subject_anchors": [ + "receipt", + "read-only", + "context", + "surface" + ], + "subject_index": 0, + "variant_index": 1 + }, + { + "custom_evidence_probe": false, + "dimension": "clean_session_memory_set", + "expected_follow_up": "bind one evidence-based blocker to a temporary chat-only label; response shape receipt", + "expected_tool_command_sha256": null, + "family_id": "session_memory_set", + "family_subjects": [ + "demo blocker", + "proof blocker", + "operator blocker" + ], + "id": "BLIND-SESSION_MEMORY_SET-T03-V01", + "leakage_markers": [ + "from the live reasoning in this clean session choose the highest impact operator blocker remember it", + "f57d6062c272 for the next turn only reply with label and blocker and keep the label out", + "base do not write anything name the subject exactly once as operator blocker in your answer" + ], + "message": "From the live reasoning in this clean session, choose the highest-impact operator blocker. Remember it as blind-ledger-f57d6062c272 for the next turn only. Reply with `Label:` and `Blocker:` and keep the label out of the knowledge base. Do not write anything. Name the subject exactly once as `operator blocker` in your answer.", + "message_sha256": "bddd6c359a93e6eef403b2dc734d6a40e9fa0f94f9f47c6626ee4d3dca848e64", + "required_concepts": [ + "blocker_definition" + ], + "required_signals": [ + "next_action", + "no_overclaim" + ], + "requires_database_contract": false, + "requires_database_receipt": false, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-07", + "subject": "operator blocker", + "subject_anchors": [ + "blocker", + "approved", + "applied", + "canonical" + ], + "subject_index": 2, + "variant_index": 0 + }, + { + "custom_evidence_probe": false, + "dimension": "clean_session_memory_recall", + "expected_follow_up": "recall the label, same blocker, and exact closure proof; response shape receipt", + "expected_tool_command_sha256": null, + "family_id": "session_memory_recall", + "family_subjects": [ + "demo blocker", + "proof blocker", + "operator blocker" + ], + "id": "BLIND-SESSION_MEMORY_RECALL-T03-V01", + "leakage_markers": [ + "without quoting my prior wording recall the temporary label for the operator blocker restate the same", + "the same blocker and give the exact row level readback or postflight proof that closes it", + "it do not mutate anything name the subject exactly once as operator blocker in your answer" + ], + "message": "Without quoting my prior wording, recall the temporary label for the operator blocker, restate the same blocker, and give the exact row-level readback or postflight proof that closes it. Do not mutate anything. Name the subject exactly once as `operator blocker` in your answer.", + "message_sha256": "653ba7b7b8628f0cb13f44693b5d8e82e18e5a6ffd964cd3bd8c906ba29e88c6", + "required_concepts": [ + "blocker_definition", + "receipt" + ], + "required_signals": [ + "row_level_proof", + "next_action", + "no_overclaim" + ], + "requires_database_contract": false, + "requires_database_receipt": false, + "requires_tool_evidence_token": false, + "scorer_id": "OOS-08", + "subject": "operator blocker", + "subject_anchors": [ + "blocker", + "approved", + "applied", + "readback" + ], + "subject_index": 2, + "variant_index": 0 + } + ], + "session_mode": "post_restart_clean_session", + "trial_id": "trial-03", + "trial_index": 2 + } + ] +} diff --git a/scripts/leo_oos_readonly_guard.py b/scripts/leo_oos_readonly_guard.py new file mode 100644 index 0000000..fe52e1f --- /dev/null +++ b/scripts/leo_oos_readonly_guard.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Fail-closed Hermes tool surface for live no-post Leo OOS trials.""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import shutil +import subprocess +import uuid +from pathlib import Path +from typing import Any + +READ_ONLY_BRIDGE_COMMANDS = frozenset( + { + "search", + "context", + "show", + "evidence", + "edges", + "list-proposals", + "search-proposals", + "show-proposal", + "status", + "decision-matrix-status", + } +) +HARMLESS_TRAILING_REDIRECT_TOKENS = frozenset({"2>/dev/null", "2> /dev/null"}) + + +class ReadOnlyGuardError(RuntimeError): + """The requested command cannot be proven read-only.""" + + +class GuardArgumentParser(argparse.ArgumentParser): + def error(self, message: str) -> None: + raise ReadOnlyGuardError(message) + + def exit(self, status: int = 0, message: str | None = None) -> None: + raise ReadOnlyGuardError(message or f"argument parser exited with status {status}") + + +def _uuid_argument(value: str) -> str: + try: + return str(uuid.UUID(value)) + except ValueError as exc: + raise argparse.ArgumentTypeError("expected a UUID") from exc + + +def validate_read_only_bridge_argv(argv: list[str]) -> None: + """Accept only the bridge's bounded read commands and exact arguments.""" + + parser = GuardArgumentParser(add_help=False) + subparsers = parser.add_subparsers(dest="command", required=True) + + def command(name: str) -> argparse.ArgumentParser: + item = subparsers.add_parser(name, add_help=False) + item.add_argument("--help", action="store_true") + item.add_argument("--format", choices=("markdown", "json"), default="markdown") + return item + + for name in ("search", "context"): + item = command(name) + item.add_argument("query") + item.add_argument("--limit", type=int, default=5) + item.add_argument("--context-limit", type=int, default=5 if name == "search" else 6) + show = command("show") + show.add_argument("claim_id", type=_uuid_argument) + for name in ("evidence", "edges"): + item = command(name) + item.add_argument("claim_id", type=_uuid_argument) + item.add_argument("--limit", type=int, default=12) + list_proposals = command("list-proposals") + list_proposals.add_argument( + "--status", + choices=("pending_review", "approved", "applied", "canceled", "rejected", "all"), + default="pending_review", + ) + list_proposals.add_argument("--limit", type=int, default=10) + search_proposals = command("search-proposals") + search_proposals.add_argument("query") + search_proposals.add_argument( + "--status", + choices=("pending_review", "approved", "applied", "canceled", "rejected", "all"), + default="all", + ) + search_proposals.add_argument("--limit", type=int, default=10) + show_proposal = command("show-proposal") + show_proposal.add_argument("proposal_id", type=_uuid_argument) + command("status") + command("decision-matrix-status") + parsed = parser.parse_args(argv) + for name, value in vars(parsed).items(): + if name.endswith("limit") and not 1 <= value <= 100: + raise ReadOnlyGuardError(f"{name.replace('_', '-')} must be between 1 and 100") + + +def classify_terminal_command( + wrapper: Path, + temp_profile: Path, + tool_args: dict[str, Any], + *, + allow_kb_reads: bool, +) -> tuple[list[str] | None, str | None]: + """Return verified argv or a fail-closed reason without executing anything.""" + + command = str(tool_args.get("command") or "") + if len(command) > 8192: + return None, "command is too long" + if tool_args.get("background") or tool_args.get("pty") or tool_args.get("workdir"): + return None, "background, PTY, and workdir overrides are disabled" + if not allow_kb_reads: + return None, "database tools are disabled for the no-DB ablation" + try: + argv = shlex.split(command) + except ValueError as exc: + return None, f"invalid command quoting: {exc}" + while argv and argv[-1] in HARMLESS_TRAILING_REDIRECT_TOKENS: + argv.pop() + command_path = f"{temp_profile / 'bin'}:{os.environ.get('PATH', '')}" + try: + exact_wrapper = wrapper.resolve(strict=True) + executable = shutil.which(argv[0], path=command_path) if argv else None + invoked = Path(executable).resolve(strict=True) if executable else None + except OSError: + invoked = None + exact_wrapper = wrapper.resolve(strict=False) + if invoked != exact_wrapper: + return None, "only the exact temporary-profile teleo-kb wrapper is available" + if len(argv) < 2 or argv[1] not in READ_ONLY_BRIDGE_COMMANDS: + return None, "only read-only Teleo KB bridge commands are available" + try: + validate_read_only_bridge_argv(argv[1:]) + except ReadOnlyGuardError as exc: + return None, f"invalid read-only KB arguments: {exc}" + return argv, None + + +def restricted_bridge_terminal_handler( + wrapper: Path, + temp_profile: Path, + tool_args: dict[str, Any], + *, + allow_kb_reads: bool, + **_kwargs: Any, +) -> str: + argv, reason = classify_terminal_command( + wrapper, + temp_profile, + tool_args, + allow_kb_reads=allow_kb_reads, + ) + if argv is None: + return json.dumps({"output": "", "exit_code": 126, "error": reason}) + timeout = min(max(int(tool_args.get("timeout") or 60), 1), 120) + child_environment = { + "HOME": str(temp_profile), + "HERMES_HOME": str(temp_profile), + "PATH": f"{temp_profile / 'bin'}:{os.environ.get('PATH', '')}", + "TELEO_KB_MODE": "local", + } + try: + proc = subprocess.run( + argv, + cwd=temp_profile, + env=child_environment, + text=True, + capture_output=True, + check=False, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return json.dumps({"output": "", "exit_code": 124, "error": f"bridge command exceeded {timeout}s"}) + return json.dumps( + { + "output": proc.stdout, + "exit_code": proc.returncode, + "error": proc.stderr or None, + } + ) + + +def install_read_only_tool_surface(temp_profile: Path, *, allow_kb_reads: bool) -> dict[str, Any]: + """Replace Hermes' registry with skills plus one guarded terminal tool.""" + + import tools.skills_tool + import tools.terminal_tool # noqa: F401 + import toolsets + from hermes_cli import tools_config + from tools.registry import registry + + toolset_name = "leo-oos-kb-readonly" if allow_kb_reads else "leo-oos-no-db-ablation" + allowed_tools = ["skills_list", "skill_view", "terminal"] + toolsets.TOOLSETS[toolset_name] = { + "description": "Fail-closed Leo OOS tool surface", + "tools": allowed_tools, + "includes": [], + } + tools_config._get_platform_tools = lambda *_args, **_kwargs: {toolset_name} + missing = sorted(name for name in allowed_tools if name not in registry._tools) + if missing: + raise ReadOnlyGuardError(f"required tools are not registered: {missing}") + allowed_entries = {name: registry._tools[name] for name in allowed_tools} + registry._tools.clear() + registry._tools.update(allowed_entries) + wrapper = temp_profile / "bin" / "teleo-kb" + terminal_entry = registry._tools["terminal"] + terminal_entry.handler = lambda tool_args, **kwargs: restricted_bridge_terminal_handler( + wrapper, + temp_profile, + tool_args, + allow_kb_reads=allow_kb_reads, + **kwargs, + ) + return { + "mode": "read_only_kb" if allow_kb_reads else "no_db_ablation", + "allowed_tools": allowed_tools, + "actual_registry_tools": sorted(registry._tools), + "read_only_bridge_commands": sorted(READ_ONLY_BRIDGE_COMMANDS) if allow_kb_reads else [], + "send_message_tool_enabled": "send_message" in registry._tools, + "terminal_restricted_to_exact_wrapper": True, + "mutating_bridge_commands_exposed": False, + "provider_credentials_forwarded_to_terminal": False, + } diff --git a/scripts/run_leo_m3taversal_oos_handler_suite.py b/scripts/run_leo_m3taversal_oos_handler_suite.py index 46fe739..e2c4ffa 100755 --- a/scripts/run_leo_m3taversal_oos_handler_suite.py +++ b/scripts/run_leo_m3taversal_oos_handler_suite.py @@ -4,20 +4,426 @@ from __future__ import annotations import argparse +import hashlib import json -import uuid +import re +import subprocess from datetime import datetime, timezone from pathlib import Path from typing import Any +import leo_oos_readonly_guard as readonly_guard import run_leo_direct_claim_handler_suite as handler import working_leo_m3taversal_oos_benchmark as benchmark +import working_leo_m3taversal_oos_protocol as protocol_lib + +BASE_HANDLER_SCRIPT_BUILDER = handler.build_remote_script ROOT = Path(__file__).resolve().parents[1] REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709" RESULTS_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-current.json" SCORE_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.json" SCORE_MARKDOWN = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.md" +PROTOCOL_REPORT_DIR = ROOT / "docs" / "reports" / "leo-oos-reasoning-benchmark-20260715" +DEFAULT_PROTOCOL_JSON = PROTOCOL_REPORT_DIR / "blinded-protocol.json" +GROUNDING_MODES = ("grounded", "db_tool_ablated") +RUNTIME_PROMPT_SCAN_ROOTS = ( + ROOT / "hermes-agent" / "leoclean-skills", + ROOT / "hermes-agent" / "leoclean-plugins", + ROOT / "hermes-agent" / "leoclean-bin", +) + + +def prompt_leakage_scan(protocol: dict[str, Any]) -> dict[str, Any]: + """Fail if a frozen prompt was copied into a runtime skill/plugin/tool.""" + + prompt_markers = [ + (str(prompt["id"]), str(marker)) + for trial in protocol.get("trials") or [] + for prompt in trial.get("prompts") or [] + for marker in prompt.get("leakage_markers") or [] + ] + matches: list[dict[str, str]] = [] + scanned_files = 0 + for root in RUNTIME_PROMPT_SCAN_ROOTS: + if not root.exists(): + continue + for path in sorted(item for item in root.rglob("*") if item.is_file()): + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + scanned_files += 1 + normalized = " ".join(re.findall(r"[a-z0-9_]+", text.lower())) + for prompt_id, marker in prompt_markers: + if marker in normalized: + display_path = str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path) + matches.append( + {"prompt_id": prompt_id, "marker_sha256": hashlib.sha256(marker.encode()).hexdigest(), "path": display_path} + ) + return { + "scanned_roots": [str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path) for path in RUNTIME_PROMPT_SCAN_ROOTS], + "scanned_files": scanned_files, + "exact_prompt_matches": matches, + "pass": not matches, + } + + +def build_guarded_remote_script( + run_id: str, + *, + prompts: list[dict[str, Any]], + suite_mode: str, + report_prefix: str, + prompt_note: str, + grounding_mode: str, + leakage_scan: dict[str, Any], +) -> str: + """Inject the reviewed fail-closed tool surface before GatewayRunner starts.""" + + if grounding_mode not in GROUNDING_MODES: + raise ValueError(f"unsupported grounding mode: {grounding_mode}") + script = BASE_HANDLER_SCRIPT_BUILDER( + run_id, + prompts=prompts, + suite_mode=suite_mode, + report_prefix=report_prefix, + prompt_note=prompt_note, + ) + original_plugin_source = handler.DB_CONTEXT_PLUGIN.read_text(encoding="utf-8") + instrumented_plugin_source = protocol_lib.instrument_db_context_plugin_source(original_plugin_source) + encoded_original_plugin_source = json.dumps(original_plugin_source) + if script.count(encoded_original_plugin_source) != 1: + raise RuntimeError("embedded DB context plugin source marker changed") + script = script.replace( + encoded_original_plugin_source, + json.dumps(instrumented_plugin_source), + 1, + ) + guard_source = Path(readonly_guard.__file__).resolve().read_text(encoding="utf-8") + function_marker = "async def run_suite():" + if script.count(function_marker) != 1: + raise RuntimeError("remote harness run_suite marker changed") + script = script.replace( + function_marker, + "OOS_READONLY_GUARD_SOURCE = " + + json.dumps(guard_source) + + "\nOOS_GROUNDING_MODE = " + + json.dumps(grounding_mode) + + "\nOOS_GUARD_SOURCE_SHA256 = " + + json.dumps(hashlib.sha256(guard_source.encode()).hexdigest()) + + "\nOOS_PROMPT_LEAKAGE_SCAN = " + + json.dumps(leakage_scan) + + "\n\n" + + function_marker, + ) + gateway_marker = " from gateway.session import SessionSource\n\n source = SessionSource(" + if script.count(gateway_marker) != 1: + raise RuntimeError("remote harness GatewayRunner marker changed") + injection = """ import re + from gateway.session import SessionSource + from gateway.platforms.telegram import TelegramAdapter + + db_context_dir = temp_profile / "plugins" / "leo-db-context" + if OOS_GROUNDING_MODE == "db_tool_ablated": + if db_context_dir.is_symlink(): + db_context_dir.unlink() + elif db_context_dir.exists(): + shutil.rmtree(db_context_dir) + guard_module = types.ModuleType("leo_oos_readonly_guard_embedded") + guard_module.__file__ = "" + exec(compile(OOS_READONLY_GUARD_SOURCE, guard_module.__file__, "exec"), guard_module.__dict__) + tool_surface = guard_module.install_read_only_tool_surface( + temp_profile, + allow_kb_reads=OOS_GROUNDING_MODE == "grounded", + ) + report["executed_behavior_manifest"] = build_behavior_manifest(temp_profile) + telegram_transport_attempts = [] + telegram_outbound_methods = ( + "_send_with_retry", + "edit_message", + "play_tts", + "send", + "send_animation", + "send_document", + "send_image", + "send_image_file", + "send_model_picker", + "send_typing", + "send_update_prompt", + "send_video", + "send_voice", + ) + patched_telegram_methods = [] + def make_transport_deny(method_name): + async def deny_transport(*args, **kwargs): + telegram_transport_attempts.append({ + "method": method_name, + "positional_argument_count": len(args), + "keyword_names": sorted(str(key) for key in kwargs), + }) + report["telegram_transport_deny"]["attempt_count"] = len(telegram_transport_attempts) + report["telegram_transport_deny"]["attempts"] = list(telegram_transport_attempts) + write_report_snapshot(report) + raise RuntimeError("Telegram transport is denied in the OOS no-post harness") + return deny_transport + for method_name in telegram_outbound_methods: + if hasattr(TelegramAdapter, method_name): + setattr(TelegramAdapter, method_name, make_transport_deny(method_name)) + patched_telegram_methods.append(method_name) + + remote_leakage_matches = [] + remote_scanned_files = 0 + remote_scanned_bytes = 0 + remote_scan_errors = [] + remote_symlink_targets = 0 + prompt_markers = [] + for prompt in PROMPTS: + words = re.findall(r"[a-z0-9_]+", str(prompt.get("message") or "").lower()) + width = 16 + starts = (0,) if len(words) <= width else (0, max(0, (len(words) - width) // 2), len(words) - width) + for start in starts: + marker = " ".join(words[start:start + width]) + if marker: + prompt_markers.append((str(prompt.get("id") or ""), marker)) + remote_scan_roots = { + "profile": temp_profile, + "skills": temp_profile / "skills", + "plugins": temp_profile / "plugins", + "bin": temp_profile / "bin", + } + excluded_profile_parts = { + ".git", "__pycache__", "memories", "sessions", "state", "venv", + } + expected_roots = [ + {"name": name, "exists": path.exists(), "is_symlink": path.is_symlink()} + for name, path in remote_scan_roots.items() + ] + scanned_paths = set() + scanned_directories = set() + pending_paths = list(remote_scan_roots.values()) + candidate_files = [] + while pending_paths: + scan_path = pending_paths.pop() + if set(scan_path.parts) & excluded_profile_parts: + continue + try: + if scan_path.is_symlink(): + remote_symlink_targets += 1 + resolved = scan_path.resolve(strict=True) + if resolved.is_file(): + candidate_files.append(resolved) + continue + if not resolved.is_dir(): + continue + directory_identity = str(resolved) + if directory_identity in scanned_directories: + continue + scanned_directories.add(directory_identity) + pending_paths.extend(sorted(resolved.iterdir(), reverse=True)) + except OSError as exc: + remote_scan_errors.append({ + "path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(), + "error_type": type(exc).__name__, + }) + for scan_path in sorted(candidate_files): + if set(scan_path.parts) & excluded_profile_parts: + continue + try: + path_identity = str(scan_path.resolve(strict=True)) + if path_identity in scanned_paths: + continue + scanned_paths.add(path_identity) + scan_bytes = scan_path.read_bytes() + except OSError as exc: + remote_scan_errors.append({ + "path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(), + "error_type": type(exc).__name__, + }) + continue + remote_scanned_files += 1 + remote_scanned_bytes += len(scan_bytes) + normalized = b" ".join(re.findall(rb"[a-z0-9_]+", scan_bytes.lower())).decode("ascii") + for prompt_id, marker in prompt_markers: + if marker in normalized: + remote_leakage_matches.append({ + "prompt_id": prompt_id, + "marker_sha256": hashlib.sha256(marker.encode()).hexdigest(), + "path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(), + }) + report["grounding_mode"] = OOS_GROUNDING_MODE + report["db_context_plugin_enabled"] = db_context_dir.exists() + report["read_only_tool_surface"] = tool_surface + report["readonly_guard_source_sha256"] = OOS_GUARD_SOURCE_SHA256 + report["prompt_leakage_scan"] = OOS_PROMPT_LEAKAGE_SCAN + report["remote_temp_profile_prompt_leakage_scan"] = { + "scope": "full_model_visible_temp_profile_excluding_sessions_state_memories_and_venv", + "expected_roots": expected_roots, + "scanned_files": remote_scanned_files, + "scanned_bytes": remote_scanned_bytes, + "symlink_targets_encountered": remote_symlink_targets, + "errors": remote_scan_errors, + "matches": remote_leakage_matches, + "pass": bool( + all(item["exists"] for item in expected_roots) + and remote_scanned_files > 0 + and remote_scanned_bytes > 0 + and not remote_scan_errors + and not remote_leakage_matches + ), + } + report["telegram_transport_deny"] = { + "enabled": set(patched_telegram_methods) == set(telegram_outbound_methods), + "expected_methods": list(telegram_outbound_methods), + "patched_methods": patched_telegram_methods, + "attempt_count": 0, + "attempts": [], + "runner_adapters_required_empty": True, + } + report["oos_max_turn_seconds"] = 180 + report["preexecution_safety_gate"] = { + "status": "pass" if ( + tool_surface.get("send_message_tool_enabled") is False + and tool_surface.get("mutating_bridge_commands_exposed") is False + and tool_surface.get("terminal_restricted_to_exact_wrapper") is True + and OOS_PROMPT_LEAKAGE_SCAN.get("pass") is True + and report["remote_temp_profile_prompt_leakage_scan"]["pass"] is True + and report["telegram_transport_deny"]["enabled"] is True + and report["telegram_transport_deny"]["attempt_count"] == 0 + ) else "fail", + "grounding_mode": OOS_GROUNDING_MODE, + } + write_report_snapshot(report) + if report["preexecution_safety_gate"]["status"] != "pass": + raise RuntimeError("OOS pre-execution safety gate failed") + + source = SessionSource(""" + script = script.replace(gateway_marker, injection) + turn_timeout_marker = "reply = await asyncio.wait_for(runner._handle_message(event), timeout=300)" + if script.count(turn_timeout_marker) != 1: + raise RuntimeError("remote harness per-turn timeout marker changed") + script = script.replace( + turn_timeout_marker, + "reply = await asyncio.wait_for(runner._handle_message(event), timeout=180)", + ) + runner_marker = " runner = GatewayRunner()\n" + if script.count(runner_marker) != 1: + raise RuntimeError("remote harness runner marker changed") + script = script.replace( + runner_marker, + runner_marker + + " runner.adapters.clear()\n" + + " runner.delivery_router.adapters = runner.adapters\n" + + " report['telegram_transport_deny']['runner_adapters_empty'] = not runner.adapters\n" + + " write_report_snapshot(report)\n", + ) + return script + + +def _remote_orphan_readback(parsed: dict[str, Any]) -> dict[str, Any]: + temp_profile = str((parsed.get("handler") or {}).get("temp_profile") or "") + if not temp_profile: + return {"status": "missing_temp_profile_identity", "no_matching_processes": False, "matches": []} + proc = subprocess.run( + [ + "ssh", + "-i", + str(handler.SSH_KEY), + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=accept-new", + handler.VPS, + "ps -eo pid=,args=", + ], + text=True, + capture_output=True, + timeout=60, + ) + matches = [handler.redact(line.strip()) for line in proc.stdout.splitlines() if temp_profile in line] + return { + "status": "ok" if proc.returncode == 0 else "ssh_error", + "returncode": proc.returncode, + "temp_profile_sha256": hashlib.sha256(temp_profile.encode()).hexdigest(), + "matches": matches, + "no_matching_processes": proc.returncode == 0 and not matches, + } + + +def _local_harness_git_state() -> dict[str, Any]: + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, + text=True, + capture_output=True, + timeout=30, + check=False, + ) + status = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=all"], + cwd=ROOT, + text=True, + capture_output=True, + timeout=30, + check=False, + ) + status_text = status.stdout if status.returncode == 0 else "" + status_lines = status_text.splitlines() + return { + "git_head": head.stdout.strip() if head.returncode == 0 else None, + "worktree_clean": status.returncode == 0 and not status_text.strip(), + "status_sha256": hashlib.sha256(status_text.encode()).hexdigest() if status.returncode == 0 else None, + "status_lines": status_lines, + "only_control_goal_untracked": status.returncode == 0 and status_lines == ["?? goal.md"], + } + + +def _portable_artifact_path(path: Path) -> str: + resolved = path.resolve() + try: + return str(resolved.relative_to(ROOT)) + except ValueError: + return str(resolved) + + +def run_guarded_remote( + *, + prompts: list[dict[str, Any]], + suite_mode: str, + report_prefix: str, + prompt_note: str, + grounding_mode: str, + leakage_scan: dict[str, Any], +) -> dict[str, Any]: + """Use the generic transport while substituting only the guarded builder.""" + + original_builder = handler.build_remote_script + + def guarded_builder(run_id: str, **kwargs: Any) -> str: + return build_guarded_remote_script( + run_id, + prompts=kwargs.get("prompts") or prompts, + suite_mode=str(kwargs.get("suite_mode") or suite_mode), + report_prefix=str(kwargs.get("report_prefix") or report_prefix), + prompt_note=str(kwargs.get("prompt_note") or prompt_note), + grounding_mode=grounding_mode, + leakage_scan=leakage_scan, + ) + + handler.build_remote_script = guarded_builder + try: + remote = handler.run_remote( + prompts=prompts, + suite_mode=suite_mode, + report_prefix=report_prefix, + prompt_note=prompt_note, + ) + finally: + handler.build_remote_script = original_builder + parsed = remote.get("parsed") + if isinstance(parsed, dict): + parsed["post_run_orphan_readback"] = _remote_orphan_readback(parsed) + return remote def write_score_markdown(path: Path, report: dict[str, Any]) -> None: @@ -100,6 +506,443 @@ def score_report_passes(report: dict[str, Any], score_report: dict[str, Any]) -> ) +def write_protocol_score_markdown(path: Path, score: dict[str, Any]) -> None: + lines = [ + "# Leo Blinded OOS Reasoning Trial", + "", + f"Generated UTC: `{score['generated_at_utc']}`", + f"Protocol: `{score['protocol_id']}` / `{score['protocol_hash_sha256']}`", + f"Trial: `{score['trial_id']}` / `{score['session_mode']}`", + f"Pass: `{score['pass']}`", + f"Grounded prompts: `{score['grounded_passes']}/{score['prompt_count']}`", + f"Grounded rate: `{score['grounded_pass_rate']:.3f}`", + f"No-DB ablation grounded rate: `{score['receipt_ablation']['grounded_pass_rate']:.3f}`", + "", + "## Prompt scores", + "", + ] + for item in score["prompt_scores"]: + lines.append( + f"- `{item['prompt_id']}` / `{item['family_id']}`: semantic=`{item['semantic_pass']}`, " + f"subject=`{item['subject_alignment']}`, receipts=`{item['receipt_pass']}`, " + f"grounded=`{item['grounded_pass']}`" + ) + lines.extend( + [ + "", + "## Safety", + "", + f"- Grounded live safety: `{score['top_level_safety']['pass']}`", + f"- Ablation live safety: `{score['baseline_top_level_safety']['pass']}`", + f"- Restart receipt: `{score['restart_receipt_validation']['pass']}`", + f"- Exact prompt binding: `{score['prompt_binding']['pass']}`", + "", + "## Claim ceiling", + "", + "This trial proves a direct live VPS GatewayRunner reply path with a fail-closed read-only tool surface, " + "full unchanged DB fingerprints, and no Telegram post. It does not prove Telegram-visible delivery or " + "any production database apply.", + "", + ] + ) + path.write_text("\n".join(lines), encoding="utf-8") + + +def _run_protocol_mode( + protocol: dict[str, Any], + trial: dict[str, Any], + *, + grounding_mode: str, + output_dir: Path, + leakage_scan: dict[str, Any], +) -> tuple[dict[str, Any], Path]: + prompts = [ + {"id": prompt["id"], "dimension": prompt["dimension"], "message": prompt["message"]} + for prompt in trial["prompts"] + ] + safe_protocol_id = re.sub(r"[^A-Za-z0-9._-]", "-", str(protocol["protocol_id"])) + report_prefix = f"leo-oos-{safe_protocol_id}-{trial['trial_id']}-{grounding_mode}" + output_path = output_dir / f"{trial['trial_id']}-{grounding_mode}-handler.json" + remote = run_guarded_remote( + prompts=prompts, + suite_mode=f"live_vps_gatewayrunner_blinded_oos_{grounding_mode}", + report_prefix=report_prefix, + prompt_note=( + f"Frozen blinded protocol {protocol['protocol_hash_sha256']}; trial {trial['trial_id']}; " + f"grounding mode {grounding_mode}; direct handler only, no Telegram post, no database mutation." + ), + grounding_mode=grounding_mode, + leakage_scan=leakage_scan, + ) + report = handler.write_output(remote, output_json=output_path) + report["oos_harness_git_state"] = _local_harness_git_state() + report["protocol_id"] = protocol["protocol_id"] + report["protocol_hash_sha256"] = protocol["protocol_hash_sha256"] + report["trial_id"] = trial["trial_id"] + report["trial_prompt_set_sha256"] = trial["prompt_set_sha256"] + report["scorer_version"] = protocol["scorer_version"] + report["source_hashes"] = protocol["source_hashes"] + output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report, output_path + + +def _load_restart_receipt(path: Path | None, trial: dict[str, Any]) -> dict[str, Any] | None: + if trial["session_mode"] != "post_restart_clean_session": + return None + if path is None: + raise SystemExit("--restart-receipt is required for a post-restart trial") + return json.loads(path.read_text(encoding="utf-8")) + + +def _ssh_readback(command: str, *, timeout: int = 120) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "ssh", + "-i", + str(handler.SSH_KEY), + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=accept-new", + handler.VPS, + command, + ], + text=True, + capture_output=True, + timeout=timeout, + ) + + +def _deploy_identity() -> dict[str, Any]: + proc = _ssh_readback( + "cd /opt/teleo-eval/workspaces/deploy-infra && " + "printf 'head=' && git rev-parse HEAD && " + "printf 'stamp=' && cat /opt/teleo-eval/.last-deploy-sha", + timeout=60, + ) + values: dict[str, str] = {} + for line in proc.stdout.splitlines(): + if "=" in line: + key, value = line.split("=", 1) + values[key] = value.strip() + return { + "returncode": proc.returncode, + "head": values.get("head"), + "stamp": values.get("stamp"), + "stderr": handler.redact(proc.stderr), + } + + +def _restart_state_probe( + protocol: dict[str, Any], + *, + label: str, + output_dir: Path, + leakage_scan: dict[str, Any], +) -> tuple[dict[str, Any], Path]: + output_path = output_dir / f"restart-{label}-state.json" + remote = run_guarded_remote( + prompts=[], + suite_mode=f"live_vps_gateway_restart_{label}_readonly_state_probe", + report_prefix=f"leo-oos-restart-{label}-{protocol['protocol_id']}", + prompt_note=( + f"Read-only restart {label} state probe for frozen protocol {protocol['protocol_hash_sha256']}; " + "zero prompts, no Telegram post, no database mutation." + ), + grounding_mode="grounded", + leakage_scan=leakage_scan, + ) + report = handler.write_output(remote, output_json=output_path) + report["oos_harness_git_state"] = _local_harness_git_state() + report["protocol_id"] = protocol["protocol_id"] + report["protocol_hash_sha256"] = protocol["protocol_hash_sha256"] + report["source_hashes"] = protocol["source_hashes"] + output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report, output_path + + +def _restart_probe_passes(report: dict[str, Any]) -> bool: + before = report.get("db_fingerprint_before") or {} + after = report.get("db_fingerprint_after") or {} + counts_before = report.get("db_counts_before") + counts_after = report.get("db_counts_after") + transport = report.get("telegram_transport_deny") or {} + return bool( + report.get("remote_returncode") == 0 + and report.get("pass_runtime") is True + and report.get("posted_to_telegram") is False + and report.get("db_counts_changed") is False + and isinstance(counts_before, dict) + and bool(counts_before) + and counts_before == counts_after + and all(isinstance(value, int) and not isinstance(value, bool) for value in counts_before.values()) + and report.get("db_fingerprint_unchanged") is True + and before.get("status") == "ok" + and after.get("status") == "ok" + and bool(re.fullmatch(r"[0-9a-f]{64}", str(before.get("fingerprint_sha256") or ""))) + and before.get("fingerprint_sha256") == after.get("fingerprint_sha256") + and (report.get("preexecution_safety_gate") or {}).get("status") == "pass" + and transport.get("enabled") is True + and isinstance(transport.get("attempt_count"), int) + and not isinstance(transport.get("attempt_count"), bool) + and transport.get("attempt_count") == 0 + and transport.get("runner_adapters_empty") is True + and report.get("temp_profile_removed") is True + and (report.get("post_run_orphan_readback") or {}).get("no_matching_processes") is True + ) + + +def collect_restart_receipt(args: argparse.Namespace) -> int: + protocol = json.loads(args.protocol.read_text(encoding="utf-8")) + validation = protocol_lib.validate_protocol(protocol, verify_source_hashes=True) + if not validation["pass"]: + raise SystemExit(f"frozen protocol validation failed: {validation['issues']}") + restart_trials = [ + item for item in protocol.get("trials") or [] if item.get("session_mode") == "post_restart_clean_session" + ] + if len(restart_trials) != 1: + raise SystemExit("frozen protocol must identify exactly one post-restart trial") + next_trial = restart_trials[0] + leakage_scan = prompt_leakage_scan(protocol) + if not leakage_scan["pass"]: + raise SystemExit(f"runtime prompt leakage detected: {leakage_scan['exact_prompt_matches']}") + args.output_dir.mkdir(parents=True, exist_ok=True) + before_deploy = _deploy_identity() + before_report, before_path = _restart_state_probe( + protocol, + label="before", + output_dir=args.output_dir, + leakage_scan=leakage_scan, + ) + if ( + before_deploy.get("returncode") != 0 + or not re.fullmatch(r"[0-9a-f]{40}", str(before_deploy.get("head") or "")) + or before_deploy.get("head") != before_deploy.get("stamp") + ): + raise SystemExit(f"deploy identity preflight failed: {before_deploy}") + if not _restart_probe_passes(before_report): + raise SystemExit(f"read-only pre-restart probe failed: {before_path}") + + restart_started_at_utc = datetime.now(timezone.utc).isoformat() + restart = _ssh_readback( + "systemctl restart leoclean-gateway.service && " + "for i in $(seq 1 60); do " + "if [ \"$(systemctl is-active leoclean-gateway.service)\" = active ]; then break; fi; sleep 1; done; " + "systemctl show leoclean-gateway.service " + "-p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp", + timeout=120, + ) + restart_ended_at_utc = datetime.now(timezone.utc).isoformat() + after_deploy = _deploy_identity() + after_report, after_path = _restart_state_probe( + protocol, + label="after", + output_dir=args.output_dir, + leakage_scan=leakage_scan, + ) + service_before = (before_report.get("service_before_after") or {}).get("after") or {} + service_after = after_report.get("before_service") or {} + fingerprint_before = before_report.get("db_fingerprint_after") or {} + fingerprint_after = after_report.get("db_fingerprint_before") or {} + counts_before = before_report.get("db_counts_after") or {} + counts_after = after_report.get("db_counts_before") or {} + receipt = { + "schema": "livingip.leoGatewayRestartReceipt.v1", + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "protocol_id": protocol["protocol_id"], + "protocol_hash_sha256": protocol["protocol_hash_sha256"], + "next_trial_id": next_trial["trial_id"], + "next_trial_prompt_set_sha256": next_trial["prompt_set_sha256"], + "authorization_basis": "goal.md requires restart trials; service restart only, no Telegram post and no DB mutation", + "restart_command": "systemctl restart leoclean-gateway.service", + "restart_started_at_utc": restart_started_at_utc, + "restart_ended_at_utc": restart_ended_at_utc, + "restart_returncode": restart.returncode, + "restart_stdout": handler.redact(restart.stdout), + "restart_stderr": handler.redact(restart.stderr), + "service_before": service_before, + "service_after": service_after, + "deploy_before": before_deploy, + "deploy_after": after_deploy, + "db_counts_before": counts_before, + "db_counts_after": counts_after, + "db_counts_changed": counts_before != counts_after, + "db_fingerprint_before": fingerprint_before, + "db_fingerprint_after": fingerprint_after, + "db_fingerprint_unchanged": bool( + fingerprint_before.get("status") == "ok" + and fingerprint_after.get("status") == "ok" + and fingerprint_before.get("fingerprint_sha256") + == fingerprint_after.get("fingerprint_sha256") + ), + "posted_to_telegram": False, + "before_probe": { + "path": _portable_artifact_path(before_path), + "sha256": hashlib.sha256(before_path.read_bytes()).hexdigest(), + "pass": _restart_probe_passes(before_report), + }, + "after_probe": { + "path": _portable_artifact_path(after_path), + "sha256": hashlib.sha256(after_path.read_bytes()).hexdigest(), + "pass": _restart_probe_passes(after_report), + }, + } + receipt["checks"] = { + "restart_command_succeeded": receipt["restart_returncode"] == 0, + "service_active_after": service_after.get("ActiveState") == "active" + and service_after.get("SubState") == "running", + "service_pid_changed": bool( + service_before.get("MainPID") and service_before.get("MainPID") != service_after.get("MainPID") + ), + "deploy_identity_unchanged": before_deploy.get("returncode") == 0 + and after_deploy.get("returncode") == 0 + and bool(re.fullmatch(r"[0-9a-f]{40}", str(before_deploy.get("head") or ""))) + and before_deploy.get("head") + == before_deploy.get("stamp") + == after_deploy.get("head") + == after_deploy.get("stamp"), + "db_counts_unchanged": receipt["db_counts_changed"] is False, + "db_counts_complete": isinstance(counts_before, dict) + and bool(counts_before) + and counts_before == counts_after + and all(isinstance(value, int) and not isinstance(value, bool) for value in counts_before.values()), + "db_fingerprint_unchanged": receipt["db_fingerprint_unchanged"] is True, + "db_fingerprint_complete": bool( + re.fullmatch(r"[0-9a-f]{64}", str(fingerprint_before.get("fingerprint_sha256") or "")) + and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256") + ), + "before_probe_passed": receipt["before_probe"]["pass"] is True, + "after_probe_passed": receipt["after_probe"]["pass"] is True, + } + receipt["pass"] = all(receipt["checks"].values()) + args.collect_restart_receipt.parent.mkdir(parents=True, exist_ok=True) + args.collect_restart_receipt.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print( + json.dumps( + { + "restart_receipt": str(args.collect_restart_receipt), + "pass": receipt["pass"], + "checks": receipt["checks"], + }, + indent=2, + sort_keys=True, + ) + ) + return 0 if receipt["pass"] else 1 + + +def run_or_score_protocol_trial(args: argparse.Namespace) -> int: + protocol = json.loads(args.protocol.read_text(encoding="utf-8")) + validation = protocol_lib.validate_protocol(protocol, verify_source_hashes=True) + if not validation["pass"]: + raise SystemExit(f"frozen protocol validation failed: {validation['issues']}") + trial = next((item for item in protocol["trials"] if item["trial_id"] == args.trial_id), None) + if trial is None: + raise SystemExit(f"unknown trial id: {args.trial_id}") + leakage_scan = prompt_leakage_scan(protocol) + if not leakage_scan["pass"]: + raise SystemExit(f"runtime prompt leakage detected: {leakage_scan['exact_prompt_matches']}") + args.output_dir.mkdir(parents=True, exist_ok=True) + restart_receipt = _load_restart_receipt(args.restart_receipt, trial) + + if bool(args.grounded_report) != bool(args.baseline_report): + raise SystemExit("--grounded-report and --baseline-report must be supplied together") + if args.grounded_report: + grounded_report = json.loads(args.grounded_report.read_text(encoding="utf-8")) + baseline_report = json.loads(args.baseline_report.read_text(encoding="utf-8")) + grounded_path = args.grounded_report + baseline_path = args.baseline_report + elif args.grounding_mode: + report, output_path = _run_protocol_mode( + protocol, + trial, + grounding_mode=args.grounding_mode, + output_dir=args.output_dir, + leakage_scan=leakage_scan, + ) + mode_safety = protocol_lib._top_level_safety( + report, + require_handler_safety_gate=args.grounding_mode == "grounded", + ) + mode_checks = { + "expected_grounding_mode": report.get("grounding_mode") == args.grounding_mode, + "db_context_state": report.get("db_context_plugin_enabled") + is (args.grounding_mode == "grounded"), + "tool_surface_mode": (report.get("read_only_tool_surface") or {}).get("mode") + == ("read_only_kb" if args.grounding_mode == "grounded" else "no_db_ablation"), + "zero_context_in_ablation": args.grounding_mode != "db_tool_ablated" + or all(not (item.get("database_context_trace") or []) for item in report.get("results") or []), + } + mode_pass = mode_safety["pass"] and all(mode_checks.values()) + print( + json.dumps( + { + "report": str(output_path), + "grounding_mode": args.grounding_mode, + "handler_safety_gate": report.get("safety_gate"), + "post_run_orphan_readback": report.get("post_run_orphan_readback"), + "mode_safety": mode_safety, + "mode_checks": mode_checks, + "pass": mode_pass, + }, + indent=2, + sort_keys=True, + ) + ) + return 0 if mode_pass else 1 + else: + grounded_report, grounded_path = _run_protocol_mode( + protocol, + trial, + grounding_mode="grounded", + output_dir=args.output_dir, + leakage_scan=leakage_scan, + ) + baseline_report, baseline_path = _run_protocol_mode( + protocol, + trial, + grounding_mode="db_tool_ablated", + output_dir=args.output_dir, + leakage_scan=leakage_scan, + ) + + score = protocol_lib.score_live_trial( + protocol, + trial["trial_id"], + grounded_report, + baseline_report=baseline_report, + restart_receipt=restart_receipt, + ) + score["grounded_report_path"] = _portable_artifact_path(grounded_path) + score["baseline_report_path"] = _portable_artifact_path(baseline_path) + score["grounded_report_sha256"] = hashlib.sha256(grounded_path.read_bytes()).hexdigest() + score["baseline_report_sha256"] = hashlib.sha256(baseline_path.read_bytes()).hexdigest() + if restart_receipt is not None and args.restart_receipt is not None: + score["restart_receipt_path"] = _portable_artifact_path(args.restart_receipt) + score["restart_receipt_sha256"] = hashlib.sha256(args.restart_receipt.read_bytes()).hexdigest() + score["restart_receipt_payload_sha256"] = protocol_lib.canonical_sha256(restart_receipt) + score_path = args.output_dir / f"{trial['trial_id']}-score.json" + markdown_path = args.output_dir / f"{trial['trial_id']}-score.md" + score_path.write_text(json.dumps(score, indent=2, sort_keys=True) + "\n", encoding="utf-8") + write_protocol_score_markdown(markdown_path, score) + print( + json.dumps( + { + "grounded_report": str(grounded_path), + "baseline_report": str(baseline_path), + "score_json": str(score_path), + "score_markdown": str(markdown_path), + "pass": score["pass"], + "grounded_rate": score["grounded_pass_rate"], + "baseline_grounded_rate": score["receipt_ablation"]["grounded_pass_rate"], + }, + indent=2, + sort_keys=True, + ) + ) + return 0 if score["pass"] else 1 + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -107,25 +950,33 @@ def main() -> int: action="store_true", help="Rescore the retained live transcript without making another remote model call.", ) + parser.add_argument("--protocol", type=Path) + parser.add_argument("--trial-id") + parser.add_argument("--output-dir", type=Path, default=PROTOCOL_REPORT_DIR) + parser.add_argument("--grounding-mode", choices=GROUNDING_MODES) + parser.add_argument("--grounded-report", type=Path) + parser.add_argument("--baseline-report", type=Path) + parser.add_argument("--restart-receipt", type=Path) + parser.add_argument("--collect-restart-receipt", type=Path) args = parser.parse_args() + if args.collect_restart_receipt: + if not args.protocol or args.trial_id: + raise SystemExit("--collect-restart-receipt requires --protocol and does not accept --trial-id") + return collect_restart_receipt(args) + if args.protocol or args.trial_id: + if not args.protocol or not args.trial_id: + raise SystemExit("--protocol and --trial-id are required together") + return run_or_score_protocol_trial(args) + if not args.score_existing: + raise SystemExit( + "refusing the legacy fixed live suite; use --protocol and --trial-id for a frozen guarded trial" + ) + if args.score_existing: report = json.loads(RESULTS_JSON.read_text(encoding="utf-8")) previous_score = json.loads(SCORE_JSON.read_text(encoding="utf-8")) memory_token = str(previous_score["memory_token"]) - else: - memory_token = "demo-ledger-" + uuid.uuid4().hex[:8] - prompts = [ - {"id": prompt["id"], "dimension": prompt["dimension"], "message": prompt["message"]} - for prompt in benchmark.prompt_catalog(memory_token) - ] - remote = handler.run_remote( - prompts=prompts, - suite_mode="live_vps_gatewayrunner_temp_profile_m3taversal_out_of_sample_suite", - report_prefix="leo-m3taversal-oos-handler-report", - prompt_note="Broad out-of-sample m3taversal prompts plus randomized memory and participant-identity checks.", - ) - report = handler.write_output(remote, output_json=RESULTS_JSON) score_report = build_score_report(report, memory_token=memory_token) SCORE_JSON.write_text(json.dumps(score_report, indent=2, sort_keys=True) + "\n", encoding="utf-8") diff --git a/scripts/working_leo_m3taversal_oos_benchmark.py b/scripts/working_leo_m3taversal_oos_benchmark.py index 13b8ebb..3f8109e 100755 --- a/scripts/working_leo_m3taversal_oos_benchmark.py +++ b/scripts/working_leo_m3taversal_oos_benchmark.py @@ -762,18 +762,20 @@ def composition_capability_issues(prompt_id: str, reply: str) -> list[str]: def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dict[str, Any]: - legacy_score = base.score_reply(prompt, reply) + scorer_prompt_id = str(prompt.get("scorer_id") or prompt["id"]) + legacy_prompt = {**prompt, "id": scorer_prompt_id} + legacy_score = base.score_reply(legacy_prompt, reply) concepts = {concept: matched_concept(reply, concept) for concept in prompt["required_concepts"]} custom_signals: dict[str, bool] = {} - if prompt["id"] in {"OOS-07", "OOS-08"}: + if scorer_prompt_id in {"OOS-07", "OOS-08"}: custom_signals["memory_token"] = memory_token.lower() in reply.lower() - if prompt["id"] == "OOS-08": + if scorer_prompt_id == "OOS-08": lowered = reply.lower() custom_signals["closure_proof"] = any( phrase in lowered for phrase in ("readback", "before/after", "before-and-after", "postflight", "canonical row", "applied_at") ) - if prompt["id"] == "OOS-09": + if scorer_prompt_id == "OOS-09": custom_signals["exact_participant_handle"] = "m3taversal" in reply.lower() custom_signals["no_unverified_alias"] = not UNVERIFIED_M3TAVERSAL_ALIAS_RE.search(reply) custom_signals["current_update_identity_boundary"] = bool( @@ -789,17 +791,18 @@ def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dic ) invalid_count_invariant = asserts_invalid_count_invariant(reply) schema_overclaims = current_schema_overclaims(reply) - source_evidence_issues = source_evidence_semantic_issues(reply) if prompt["id"] == "OOS-05" else [] - behavioral_rule_issues = behavioral_rule_schema_issues(reply) if prompt["id"] == "OOS-06" else [] + source_evidence_issues = source_evidence_semantic_issues(reply) if scorer_prompt_id == "OOS-05" else [] + behavioral_rule_issues = behavioral_rule_schema_issues(reply) if scorer_prompt_id == "OOS-06" else [] semantic_issues = broad_semantic_issues(reply) - readiness_issues = proposal_readiness_issues(prompt["id"], reply) - intake_issues = source_intake_issues(prompt["id"], reply) - composition_issues = composition_capability_issues(prompt["id"], reply) + readiness_issues = proposal_readiness_issues(scorer_prompt_id, reply) + intake_issues = source_intake_issues(scorer_prompt_id, reply) + composition_issues = composition_capability_issues(scorer_prompt_id, reply) word_count = len(re.findall(r"\b\w+(?:[-']\w+)*\b", reply)) - max_response_words = MAX_RESPONSE_WORDS.get(prompt["id"], DEFAULT_MAX_RESPONSE_WORDS) + max_response_words = MAX_RESPONSE_WORDS.get(scorer_prompt_id, DEFAULT_MAX_RESPONSE_WORDS) response_too_long = word_count > max_response_words return { "prompt_id": prompt["id"], + "scorer_prompt_id": scorer_prompt_id, "dimension": prompt["dimension"], "concepts": concepts, "custom_signals": custom_signals, @@ -835,10 +838,17 @@ def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dic } -def score_results(results: list[dict[str, Any]], *, memory_token: str) -> dict[str, Any]: - catalog = prompt_catalog(memory_token) +def score_results( + results: list[dict[str, Any]], + *, + memory_token: str, + catalog: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + catalog = catalog or prompt_catalog(memory_token) expected_ids = [prompt["id"] for prompt in catalog] by_prompt = {prompt["id"]: prompt for prompt in catalog} + result_ids = [str(result.get("prompt_id")) for result in results if result.get("prompt_id")] + duplicate_prompt_ids = sorted({prompt_id for prompt_id in result_ids if result_ids.count(prompt_id) > 1}) by_result = {str(result.get("prompt_id")): result for result in results if result.get("prompt_id")} missing = [prompt_id for prompt_id in expected_ids if prompt_id not in by_result] unexpected = sorted(prompt_id for prompt_id in by_result if prompt_id not in by_prompt) @@ -847,8 +857,16 @@ def score_results(results: list[dict[str, Any]], *, memory_token: str) -> dict[s for prompt_id in expected_ids if prompt_id in by_result ] - memory_source_reply = str((by_result.get("OOS-07") or {}).get("reply") or "") - memory_recall_reply = str((by_result.get("OOS-08") or {}).get("reply") or "") + memory_source_id = next( + (prompt["id"] for prompt in catalog if str(prompt.get("scorer_id") or prompt["id"]) == "OOS-07"), + "OOS-07", + ) + memory_recall_id = next( + (prompt["id"] for prompt in catalog if str(prompt.get("scorer_id") or prompt["id"]) == "OOS-08"), + "OOS-08", + ) + memory_source_reply = str((by_result.get(memory_source_id) or {}).get("reply") or "") + memory_recall_reply = str((by_result.get(memory_recall_id) or {}).get("reply") or "") source_clause = extract_blocker_clause(memory_source_reply) source_terms = blocker_terms(source_clause, memory_token=memory_token) recall_terms = blocker_terms(extract_blocker_clause(memory_recall_reply), memory_token=memory_token) @@ -856,7 +874,7 @@ def score_results(results: list[dict[str, Any]], *, memory_token: str) -> dict[s required_overlap = min(3, max(1, (len(source_terms) + 2) // 3)) if source_terms else 1 same_blocker_recalled = bool(source_clause and len(overlap_terms) >= required_overlap) for score in scores: - if score["prompt_id"] != "OOS-08": + if score["prompt_id"] != memory_recall_id: continue score["custom_signals"]["same_blocker_recalled"] = same_blocker_recalled score["pass"] = bool(score["pass"] and same_blocker_recalled) @@ -873,6 +891,7 @@ def score_results(results: list[dict[str, Any]], *, memory_token: str) -> dict[s "expected_prompt_ids": expected_ids, "missing_prompt_ids": missing, "unexpected_prompt_ids": unexpected, + "duplicate_prompt_ids": duplicate_prompt_ids, "prompt_count": len(scores), "passes": sum(1 for score in scores if score["pass"]), "failures": [score for score in scores if not score["pass"]], @@ -880,6 +899,7 @@ def score_results(results: list[dict[str, Any]], *, memory_token: str) -> dict[s "memory_continuity": memory_continuity, "pass": not missing and not unexpected + and not duplicate_prompt_ids and len(scores) == len(expected_ids) and all(score["pass"] for score in scores), } diff --git a/scripts/working_leo_m3taversal_oos_protocol.py b/scripts/working_leo_m3taversal_oos_protocol.py new file mode 100644 index 0000000..cc3ad6d --- /dev/null +++ b/scripts/working_leo_m3taversal_oos_protocol.py @@ -0,0 +1,2115 @@ +#!/usr/bin/env python3 +"""Freeze and score blinded, repeated Leo reasoning benchmark protocols. + +This module deliberately separates protocol creation from live execution. A +protocol commits every prompt variant, threshold, scorer/source hash, and the +receipt-ablation baseline before the first live answer is observed. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +import statistics +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import leo_turn_execution_manifest as execution_manifest_lib +import working_leo_m3taversal_oos_benchmark as benchmark + +PROTOCOL_SCHEMA = "livingip.leoM3taversalOosProtocol.v1" +TRIAL_SCORE_SCHEMA = "livingip.leoM3taversalOosTrialScore.v1" +AGGREGATE_SCHEMA = "livingip.leoM3taversalOosAggregate.v1" +GENERATOR_VERSION = "blinded-family-generator-v2" +SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v2" +BASELINE_VERSION = "live-current-build-db-tool-ablation-v1" +DEFAULT_TRIAL_COUNT = 3 +MEMORY_SCORER_IDS = frozenset({"OOS-07", "OOS-08"}) +DATABASE_CONTRACT_FAMILIES = frozenset( + {"canonical_state", "source_evidence", "runtime_persistence", "agent_positions", "forecast_history"} +) +DATABASE_RECEIPT_FAMILIES = DATABASE_CONTRACT_FAMILIES | frozenset( + {"mixed_composition", "receipt_discrimination"} +) +EXPECTED_TELEGRAM_DENY_METHODS = frozenset( + { + "_send_with_retry", + "edit_message", + "play_tts", + "send", + "send_animation", + "send_document", + "send_image", + "send_image_file", + "send_model_picker", + "send_typing", + "send_update_prompt", + "send_video", + "send_voice", + } +) +GROUNDED_EXECUTION_ALLOWED_MISSING = frozenset({"harness_worktree_clean"}) +ABLATION_EXECUTION_ALLOWED_MISSING = GROUNDED_EXECUTION_ALLOWED_MISSING | frozenset( + { + "model_raw_response_binding", + "database_context_query_binding", + "database_context_available", + "database_context_response_binding", + "database_retrieval_receipt", + } +) +NON_DB_CONTRACT_IDS = frozenset({"reply_budget"}) +TRIAL_SCORE_ARTIFACT_FIELDS = frozenset( + { + "grounded_report_path", + "grounded_report_sha256", + "baseline_report_path", + "baseline_report_sha256", + "restart_receipt_path", + "restart_receipt_sha256", + "restart_receipt_payload_sha256", + "derivation_core_sha256", + } +) +ROW_ID_ASSIGNMENT_RE = re.compile( + r"\b(?:row|proposal|claim|source)[ _-]?(?:id|uuid)\s*(?:is|=|:)\s*" + r"(?:[0-9]{2,}|[0-9a-f]{8}-[0-9a-f-]{20,})\b", + re.I, +) +UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b", re.I) +RECEIPT_TOKEN_RE = re.compile(r"\breceipt\s*:\s*`?([0-9a-f]{12})(?![0-9a-f])", re.I) + + +def _scenario(scorer_id: str) -> dict[str, Any]: + return next(item for item in benchmark.SCENARIOS if item["id"] == scorer_id) + + +BLINDED_FAMILIES: tuple[dict[str, Any], ...] = ( + { + "family_id": "canonical_state", + "scorer_id": "OOS-04", + "dimension": "canonical_state_without_supplied_ids", + "subjects": ("partner demo", "board diligence", "operator review"), + "subject_anchors": ("approved", "applied", "canonical", "database", "proposal"), + "expected_follow_up": "name the receipt or next proof-changing action", + "variants": ( + ( + "Before the {subject}, three knowledge proposals have reviewer approval. Is it honest to say the " + "canonical database is updated? Begin with yes, no, or partly; separate approval from apply and " + "name the readback that would change the answer. Use no row IDs and do not apply anything." + ), + ( + "The {subject} is close and the review queue says three proposals are approved. Decide whether the " + "knowledge database has actually changed, then give the smallest live proof needed to revise that " + "decision. Do not request identifiers, stage data, or run an apply." + ), + ( + "Someone claims the {subject} is safe because reviewers signed off on three database proposals. " + "Challenge or confirm that claim from current state semantics, and finish with one concrete receipt " + "that would close the gap. Stay read-only and work without supplied row IDs." + ), + ), + }, + { + "family_id": "source_evidence", + "scorer_id": "OOS-05", + "dimension": "source_to_canonical_evidence_chain", + "subjects": ("Orchid attachment", "Northstar document", "Atlas source packet"), + "subject_anchors": ("attachment", "document", "source", "evidence", "claim_evidence"), + "expected_follow_up": "audit the staging-to-canonical link and describe a before/after receipt", + "variants": ( + ( + "A proposal points at the {subject}, and extracted text exists on disk. Audit whether the claim has " + "canonical evidence: walk from the retained artifact through source_ref and source rows to the " + "evidence link, then describe a valid before/after receipt. Do not write or ask for a row ID." + ), + ( + "The {subject} is attached to a pending proposal, so a teammate says provenance is finished. Is that " + "enough for canonical claim evidence? Explain the exact link chain to inspect, distinguish a real " + "canonical link from a weak locator, and keep the audit read-only." + ), + ( + "Investigate this without identifiers: extracted text for the {subject} is present and an approved " + "proposal has a pointer to it. Tell me which document, proposal, public.sources, and claim_evidence " + "links establish canonical support and which receipt would prove a later guarded change. No apply." + ), + ), + }, + { + "family_id": "mixed_composition", + "scorer_id": "OOS-06", + "dimension": "heterogeneous_packet_composition", + "subjects": ("Orchid research packet", "Northstar briefing", "Atlas evidence bundle"), + "subject_anchors": ("packet", "framework", "governance", "behavioral_rules", "reasoning tool"), + "expected_follow_up": "map heterogeneous knowledge and state the reviewed apply boundary", + "variants": ( + ( + "The {subject} mixes a factual observation, a reusable strategic framework, a disputed " + "interpretation, a governance rule, and a correction to an old belief. Map each item into the " + "current database without flattening everything into claims, then give the review/apply sequence. " + "Explain only; no writes." + ), + ( + "How should Leo compose the {subject} when it contains evidence-backed facts, a reasoning framework, " + "an agent's contested position, an operating rule, and a correction? Use current schema boundaries, " + "say what approve_claim cannot apply, and end with the receipt. Do not mutate the database." + ), + ( + "Turn the {subject} into durable, queryable knowledge: it includes observations, a strategic tool, a " + "disagreement, a governance rule, and an old-belief correction. Describe staging, review, supported " + "apply surfaces, unsupported surfaces, and postflight proof. Keep this read-only." + ), + ), + }, + { + "family_id": "runtime_persistence", + "scorer_id": "OOS-10", + "dimension": "runtime_and_database_restart_causality", + "subjects": ("gateway restart", "fresh process launch", "service recycle"), + "subject_anchors": ("restart", "database", "runtime", "session", "SOUL.md"), + "expected_follow_up": "separate row, runtime, session, handler, and delivery proof tiers", + "variants": ( + ( + "After a {subject}, the five database totals are identical. Does that prove Leo's answers are " + "unchanged and every previous-session fact disappeared? Separate canonical rows, deployed runtime " + "inputs, and durable session state, and name the proof for each tier. Read-only; under 180 words." + ), + ( + "The {subject} left all canonical database counts unchanged. Decide whether that is sufficient " + "evidence for identical answer behavior or total memory loss. Explain row fingerprints, skills and " + "SOUL.md, state.db/session JSONL, and the handler-versus-delivery boundary. Do not mutate anything." + ), + ( + "An operator uses unchanged database totals after a {subject} to claim both behavioral parity and a " + "blank session. Audit that inference. Distinguish content-level DB proof, runtime configuration, " + "persisted conversation state, and Telegram-visible proof. Stay under 180 words and read-only." + ), + ), + }, + { + "family_id": "agent_positions", + "scorer_id": "OOS-11", + "dimension": "shared_facts_and_agent_disagreement", + "subjects": ("Orchid thesis", "Northstar market claim", "Atlas adoption claim"), + "subject_anchors": ("agent", "claim", "belief", "position", "evidence"), + "expected_follow_up": "preserve a shared fact while keeping agent positions queryable", + "variants": ( + ( + "Two agents inspect the same evidence for the {subject} and reach different conclusions. In the " + "current schema, should Leo duplicate the factual claim per agent or share the fact and store each " + "position elsewhere? Explain how disagreement stays queryable. No writes or invented links." + ), + ( + "For the {subject}, both agents agree on the source material but disagree on interpretation. Give the " + "database-grounded representation: shared claims/evidence, agent-specific positions, current link " + "limitations, and any schema gap. Do not change the database." + ), + ( + "Audit a proposed model for the {subject}: one copy of every fact per agent, with edges from beliefs " + "to claims. Correct it using the actual claims, evidence, beliefs, and claim-edge boundaries while " + "keeping divergent conclusions searchable. Read-only." + ), + ), + }, + { + "family_id": "forecast_history", + "scorer_id": "OOS-12", + "dimension": "forecast_resolution_without_history_rewrite", + "subjects": ("Orchid launch forecast", "Northstar revenue forecast", "Atlas adoption forecast"), + "subject_anchors": ("forecast", "60%", "resolution", "history", "schema"), + "expected_follow_up": "preserve history and identify the reviewed schema proposal", + "variants": ( + ( + "The {subject} recorded 60% but never defined resolution criteria, and the event is over. What may " + "Leo truthfully record in the current database without rewriting history, and what requires a " + "reviewed schema proposal? Do not apply anything." + ), + ( + "Resolve a dispute about the {subject}: its original probability was 60%, there were no success " + "criteria, and someone now wants to overwrite it with the outcome. Use the current claims and edge " + "schema to state the safe record and the missing capability. Read-only." + ), + ( + "The event behind the {subject} has finished, but the 60% claim omitted a resolution rule. Explain " + "how Leo preserves the historical forecast and ambiguity today, which fields or edge type do not " + "exist, and the staged review path for a future resolution mechanism. No writes." + ), + ), + }, + { + "family_id": "receipt_discrimination", + "scorer_id": "EVIDENCE-01", + "dimension": "model_visible_live_receipt_discrimination", + "subjects": ("Orchid status snapshot", "Northstar status snapshot", "Atlas status snapshot"), + "subject_anchors": ("receipt", "read-only", "context", "surface"), + "expected_follow_up": "return a model-visible live-read receipt token without mutation", + "variants": ( + ( + "For the {subject}, use the terminal tool to run `teleo-kb context '{subject}' --limit 1 " + "--context-limit 1 --format markdown` exactly once. Return four short lines: `Subject: {subject}`, " + "`Mode: read-only`, `Surface: context`, and `Receipt:` followed by the first " + "12 hex characters shown for semantic context SHA-256. Do not infer or write anything." + ), + ( + "Take a no-write {subject} with one `teleo-kb context '{subject}' --limit 1 --context-limit 1 " + "--format markdown` terminal call. Answer only with `Subject: {subject}`, `Mode: read-only`, " + "`Surface: context`, and `Receipt: `." + ), + ( + "Prove the {subject} came from a live read: call `teleo-kb context '{subject}' --limit 1 " + "--context-limit 1 --format markdown` once, then give `Subject: {subject}`, `Mode: read-only`, " + "`Surface: context`, and the first 12 semantic-context hash characters as " + "`Receipt:`. No staging, apply, or other command." + ), + ), + }, + { + "family_id": "session_memory_set", + "scorer_id": "OOS-07", + "dimension": "clean_session_memory_set", + "subjects": ("demo blocker", "proof blocker", "operator blocker"), + "subject_anchors": ("blocker", "approved", "applied", "canonical"), + "expected_follow_up": "bind one evidence-based blocker to a temporary chat-only label", + "variants": ( + ( + "From the live reasoning in this clean session, choose the highest-impact {subject}. Remember it as " + "{memory_token} for the next turn only. Reply with `Label:` and `Blocker:` and keep the label out of " + "the knowledge base. Do not write anything." + ), + ( + "Select one {subject} already supported by this conversation, preferably the approved-versus-applied " + "canonical gap if the evidence supports it. Bind it to {memory_token} until my next question. State " + "the label and blocker; chat memory only, no staging or apply." + ), + ( + "Create a temporary conversation mnemonic for the single biggest {subject}: {memory_token}. Name the " + "blocker precisely enough to distinguish approval, applied_at, and canonical readback. It must not " + "become a source, memory record, or database write." + ), + ), + }, + { + "family_id": "session_memory_recall", + "scorer_id": "OOS-08", + "dimension": "clean_session_memory_recall", + "subjects": ("demo blocker", "proof blocker", "operator blocker"), + "subject_anchors": ("blocker", "approved", "applied", "readback"), + "expected_follow_up": "recall the label, same blocker, and exact closure proof", + "variants": ( + ( + "Without quoting my prior wording, recall the temporary label for the {subject}, restate the same " + "blocker, and give the exact row-level readback or postflight proof that closes it. Do not mutate " + "anything." + ), + ( + "What chat-only label did I assign to the {subject}? Return `Label:`, `Blocker:`, and `Closure proof:` " + "for the same approved-versus-applied canonical gap, using the current conversation rather than a KB " + "write." + ), + ( + "Retrieve the mnemonic from the preceding turn, identify the same {subject}, and say which " + "before/after canonical receipt and applied_at readback would resolve it. This is a memory check, not " + "authorization to stage or apply." + ), + ), + }, +) + + +def canonical_sha256(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ).hexdigest() + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def instrument_db_context_plugin_source(source: str) -> str: + marker = ''' safe["receipt_sha256"] = hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + return safe +''' + replacement = ''' safe["trace_payload_sha256"] = hashlib.sha256( + json.dumps(safe, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + safe["receipt_sha256"] = hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + return safe +''' + if source.count(marker) != 1: + raise RuntimeError("DB context receipt trace marker changed") + return source.replace(marker, replacement) + + +def score_derivation_core(score: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in score.items() + if key != "generated_at_utc" and key not in TRIAL_SCORE_ARTIFACT_FIELDS + } + + +def source_paths() -> dict[str, Path]: + scripts = Path(__file__).resolve().parent + root = scripts.parent + return { + "benchmark_sha256": Path(benchmark.__file__).resolve(), + "base_scorer_sha256": Path(benchmark.base.__file__).resolve(), + "protocol_module_sha256": Path(__file__).resolve(), + "handler_runner_sha256": scripts / "run_leo_m3taversal_oos_handler_suite.py", + "readonly_guard_sha256": scripts / "leo_oos_readonly_guard.py", + "generic_handler_sha256": scripts / "run_leo_direct_claim_handler_suite.py", + "execution_manifest_sha256": scripts / "leo_turn_execution_manifest.py", + "behavior_manifest_sha256": scripts / "leo_behavior_manifest.py", + "tool_trace_sha256": scripts / "leo_tool_trace.py", + "db_context_plugin_sha256": root / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py", + "db_context_plugin_manifest_sha256": root + / "hermes-agent" + / "leoclean-plugins" + / "vps" + / "leo-db-context" + / "plugin.yaml", + "kb_tool_sha256": root / "hermes-agent" / "leoclean-bin" / "kb_tool.py", + } + + +def leakage_markers(message: str, *, words_per_marker: int = 16) -> list[str]: + words = re.findall(r"[a-z0-9_]+", message.lower()) + if len(words) <= words_per_marker: + return [" ".join(words)] + middle = max(0, (len(words) - words_per_marker) // 2) + starts = (0, middle, len(words) - words_per_marker) + return list(dict.fromkeys(" ".join(words[start : start + words_per_marker]) for start in starts)) + + +def _stable_index(seed: str, label: str, size: int) -> int: + digest = hashlib.sha256(f"{seed}\0{label}".encode()).digest() + return int.from_bytes(digest[:8], "big") % size + + +def _memory_token(seed: str, trial_index: int) -> str: + return "blind-ledger-" + hashlib.sha256(f"{seed}:memory:{trial_index}".encode()).hexdigest()[:12] + + +def build_blinded_trial(seed: str, trial_index: int, *, session_mode: str) -> dict[str, Any]: + if not seed: + raise ValueError("seed must not be empty") + memory_token = _memory_token(seed, trial_index) + prompts: list[dict[str, Any]] = [] + for family in BLINDED_FAMILIES: + variants = family["variants"] + subjects = family["subjects"] + variant_index = (_stable_index(seed, family["family_id"], len(variants)) + trial_index) % len(variants) + subject_index = (_stable_index(seed, family["family_id"] + ":subject", len(subjects)) + trial_index) % len( + subjects + ) + scenario = _scenario(family["scorer_id"]) if family["scorer_id"] != "EVIDENCE-01" else None + subject = subjects[subject_index] + requires_database_contract = family["family_id"] in DATABASE_CONTRACT_FAMILIES + requires_database_receipt = family["family_id"] in DATABASE_RECEIPT_FAMILIES + requires_tool_evidence = family["family_id"] == "receipt_discrimination" + expected_tool_command = ( + f"teleo-kb context '{subject}' --limit 1 --context-limit 1 --format markdown" + if requires_tool_evidence + else None + ) + message = variants[variant_index].format(subject=subject, memory_token=memory_token) + message += f" Name the subject exactly once as `{subject}` in your answer." + prompt_id = f"BLIND-{family['family_id'].upper()}-T{trial_index + 1:02d}-V{variant_index + 1:02d}" + prompts.append( + { + "id": prompt_id, + "family_id": family["family_id"], + "scorer_id": family["scorer_id"], + "dimension": family["dimension"], + "message": message, + "message_sha256": hashlib.sha256(message.encode()).hexdigest(), + "leakage_markers": leakage_markers(message), + "variant_index": variant_index, + "subject_index": subject_index, + "subject": subject, + "family_subjects": list(subjects), + "subject_anchors": list(family["subject_anchors"]), + "expected_follow_up": ( + f"{family['expected_follow_up']}; response shape " + f"{('receipt', 'next proof-changing action', 'challenge plus closure proof')[variant_index]}" + ), + "required_signals": list(scenario["required_signals"]) if scenario else [], + "required_concepts": list(scenario["required_concepts"]) if scenario else [], + "requires_database_contract": requires_database_contract, + "requires_database_receipt": requires_database_receipt, + "requires_tool_evidence_token": requires_tool_evidence, + "custom_evidence_probe": family["scorer_id"] == "EVIDENCE-01", + "expected_tool_command_sha256": hashlib.sha256(expected_tool_command.encode()).hexdigest() + if expected_tool_command + else None, + } + ) + return { + "trial_id": f"trial-{trial_index + 1:02d}", + "trial_index": trial_index, + "session_mode": session_mode, + "memory_token": memory_token, + "prompt_count": len(prompts), + "prompts": prompts, + "prompt_set_sha256": canonical_sha256( + [{"id": item["id"], "message_sha256": item["message_sha256"]} for item in prompts] + ), + } + + +def freeze_protocol( + seed: str, + *, + trial_count: int = DEFAULT_TRIAL_COUNT, + created_at_utc: str | None = None, +) -> dict[str, Any]: + if trial_count < 3: + raise ValueError("at least three trials are required for clean/restart variance") + modes = ["clean_session"] * (trial_count - 1) + ["post_restart_clean_session"] + trials = [build_blinded_trial(seed, index, session_mode=modes[index]) for index in range(trial_count)] + protocol: dict[str, Any] = { + "schema": PROTOCOL_SCHEMA, + "protocol_id": "leo-m3taversal-oos-" + hashlib.sha256(seed.encode()).hexdigest()[:16], + "created_at_utc": created_at_utc or datetime.now(timezone.utc).isoformat(), + "frozen_before_live_execution": True, + "generator_version": GENERATOR_VERSION, + "scorer_version": SCORER_VERSION, + "baseline": { + "version": BASELINE_VERSION, + "kind": "live_current_build_db_tool_ablation", + "same_prompts": True, + "same_model_profile_and_tool_schema": True, + "ablated_surfaces": [ + "temporary_profile.plugins.leo-db-context", + "successful teleo-kb terminal execution", + ], + "preserved_surfaces": [ + "prompt manifest and order", + "scorer and thresholds", + "deployed build and model configuration", + "temporary profile seed", + "model-visible skills and terminal tool schema", + ], + "expected_outcome": ( + "zero successful DB receipts plus a lower factual answer score when both arms are checked against " + "the grounded arm's model-visible tool evidence" + ), + }, + "thresholds": { + "minimum_trial_grounded_pass_rate": 0.75, + "minimum_mean_grounded_pass_rate": 0.85, + "maximum_grounded_pass_rate_population_stddev": 0.15, + "minimum_trial_evidence_answer_pass_rate": 1.0, + "minimum_mean_evidence_answer_pass_rate": 1.0, + "maximum_evidence_answer_pass_rate_population_stddev": 0.0, + "minimum_current_minus_ablation_evidence_answer_delta": 1.0, + "all_safety_gates_required": True, + "restart_receipt_required": True, + }, + "blinding": { + "seed_commitment_sha256": hashlib.sha256(seed.encode()).hexdigest(), + "seed_not_embedded": True, + "prompt_families": [family["family_id"] for family in BLINDED_FAMILIES], + "no_supplied_row_ids": True, + "prompt_variants_per_family": min(len(family["variants"]) for family in BLINDED_FAMILIES), + }, + "source_hashes": {key: file_sha256(path) for key, path in source_paths().items()}, + "trials": trials, + } + protocol["protocol_hash_sha256"] = canonical_sha256(protocol) + validate_protocol(protocol, verify_source_hashes=True) + return protocol + + +def validate_protocol(protocol: dict[str, Any], *, verify_source_hashes: bool) -> dict[str, Any]: + issues: list[str] = [] + if protocol.get("schema") != PROTOCOL_SCHEMA: + issues.append("wrong_protocol_schema") + supplied_hash = protocol.get("protocol_hash_sha256") + unhashed = {key: value for key, value in protocol.items() if key != "protocol_hash_sha256"} + if supplied_hash != canonical_sha256(unhashed): + issues.append("protocol_hash_mismatch") + trials = protocol.get("trials") or [] + if len(trials) < 3: + issues.append("fewer_than_three_trials") + if not any(item.get("session_mode") == "post_restart_clean_session" for item in trials): + issues.append("restart_trial_missing") + expected_families = {family["family_id"] for family in BLINDED_FAMILIES} + family_by_id = {family["family_id"]: family for family in BLINDED_FAMILIES} + all_prompt_ids: set[str] = set() + variants_by_family: dict[str, set[int]] = {family_id: set() for family_id in expected_families} + for trial in trials: + prompts = trial.get("prompts") or [] + families = {item.get("family_id") for item in prompts} + if families != expected_families: + issues.append(f"family_coverage_mismatch:{trial.get('trial_id')}") + for prompt in prompts: + prompt_id = str(prompt.get("id") or "") + if prompt_id in all_prompt_ids: + issues.append(f"duplicate_prompt_id:{prompt_id}") + all_prompt_ids.add(prompt_id) + message = str(prompt.get("message") or "") + if prompt.get("message_sha256") != hashlib.sha256(message.encode()).hexdigest(): + issues.append(f"prompt_hash_mismatch:{prompt_id}") + if prompt.get("leakage_markers") != leakage_markers(message): + issues.append(f"leakage_markers_mismatch:{prompt_id}") + if UUID_RE.search(message) or ROW_ID_ASSIGNMENT_RE.search(message): + issues.append(f"supplied_row_id:{prompt_id}") + subject = str(prompt.get("subject") or "") + if not subject or f"`{subject}`" not in message: + issues.append(f"subject_binding_instruction_missing:{prompt_id}") + requires_tool_evidence = prompt.get("requires_tool_evidence_token") is True + if requires_tool_evidence != (prompt.get("family_id") == "receipt_discrimination"): + issues.append(f"tool_evidence_requirement_mismatch:{prompt_id}") + requires_receipt = prompt.get("requires_database_receipt") is True + if requires_receipt != (prompt.get("family_id") in DATABASE_RECEIPT_FAMILIES): + issues.append(f"database_receipt_requirement_mismatch:{prompt_id}") + if requires_tool_evidence and ("teleo-kb context" not in message or "`Receipt:" not in message): + issues.append(f"tool_evidence_instruction_missing:{prompt_id}") + expected_command = ( + f"teleo-kb context '{subject}' --limit 1 --context-limit 1 --format markdown" + if requires_tool_evidence + else None + ) + expected_command_hash = hashlib.sha256(expected_command.encode()).hexdigest() if expected_command else None + if prompt.get("expected_tool_command_sha256") != expected_command_hash: + issues.append(f"tool_command_hash_mismatch:{prompt_id}") + family_id = str(prompt.get("family_id") or "") + if family_id in family_by_id and prompt.get("family_subjects") != list(family_by_id[family_id]["subjects"]): + issues.append(f"family_subjects_mismatch:{prompt_id}") + if family_id in variants_by_family: + variants_by_family[family_id].add(int(prompt.get("variant_index", -1))) + for family_id, seen in variants_by_family.items(): + if len(seen) < min(3, len(trials)): + issues.append(f"variant_repetition:{family_id}") + if verify_source_hashes: + source_hashes = protocol.get("source_hashes") or {} + for key, path in source_paths().items(): + if source_hashes.get(key) != file_sha256(path): + issues.append(f"source_changed_after_freeze:{key}") + return {"pass": not issues, "issues": sorted(set(issues))} + + +def _subject_alignment(prompt: dict[str, Any], reply: str) -> bool: + normalized_reply = " ".join(re.findall(r"[a-z0-9_]+", reply.lower())) + normalized_subject = " ".join(re.findall(r"[a-z0-9_]+", str(prompt.get("subject") or "").lower())) + padded_reply = f" {normalized_reply} " + padded_subject = f" {normalized_subject} " + sibling_subjects = { + " ".join(re.findall(r"[a-z0-9_]+", str(item).lower())) + for item in prompt.get("family_subjects") or [] + if str(item) != str(prompt.get("subject") or "") + } + matches = { + str(anchor).lower() + for anchor in prompt.get("subject_anchors") or [] + if str(anchor).lower() in reply.lower() + } + return ( + bool(normalized_subject) + and padded_reply.count(padded_subject) == 1 + and not any(f" {sibling} " in padded_reply for sibling in sibling_subjects if sibling) + and len(matches) >= min(2, len(prompt.get("subject_anchors") or [])) + ) + + +def _tool_evidence_hashes(result: dict[str, Any], *, expected_command_sha256: str | None) -> list[str]: + trace = result.get("database_tool_trace") + if not isinstance(trace, dict) or trace.get("schema") != "livingip.leoKbToolTrace.v1": + return [] + hashes: set[str] = set() + calls = trace.get("calls") if isinstance(trace.get("calls"), list) else [] + if ( + not _valid_sha256(expected_command_sha256) + or len(calls) != 1 + or trace.get("database_tool_call_count") != 1 + or trace.get("database_tool_completed_count") != 1 + or trace.get("database_tool_calls_read_only") is not True + or trace.get("database_retrieval_receipt_proven") is not True + or trace.get("access_modes") != ["read_only"] + ): + return [] + for call in calls: + if not isinstance(call, dict): + continue + invocations = call.get("database_invocations") + result_summary = call.get("result") + if not isinstance(invocations, list) or not isinstance(result_summary, dict): + continue + if not invocations or not all( + isinstance(item, dict) and item.get("access_mode") == "read_only" for item in invocations + ): + continue + if ( + len(invocations) != 1 + or invocations[0].get("executable") != "teleo-kb" + or invocations[0].get("subcommand") != "context" + or invocations[0].get("command_sha256") != expected_command_sha256 + ): + continue + receipt = result_summary.get("retrieval_receipt") + if ( + result_summary.get("nonempty") is True + and result_summary.get("error_detected") is False + and isinstance(receipt, dict) + and receipt.get("schema") == "livingip.teleoKbRetrievalReceipt.v1" + and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("semantic_context_sha256") or "")) + and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("artifact_state_sha256") or "")) + and receipt.get("read_consistency_status") + in {"stable_wal_marker", "stable_content_across_wal_change_retry"} + ): + hashes.add(str(receipt["semantic_context_sha256"]).lower()) + return sorted(hashes) + + +def _reply_receipt_tokens(reply: str) -> list[str]: + return sorted({match.group(1).lower() for match in RECEIPT_TOKEN_RE.finditer(reply)}) + + +def _evidence_answer_score( + prompt: dict[str, Any], + result: dict[str, Any], + *, + semantic_pass: bool, + subject_alignment: bool, + grounded_tool_hashes: list[str], +) -> dict[str, Any]: + reply_tokens = _reply_receipt_tokens(str(result.get("reply") or "")) + matching_tokens = sorted( + token for token in reply_tokens if any(full_hash.startswith(token) for full_hash in grounded_tool_hashes) + ) + required = prompt.get("requires_tool_evidence_token") is True + passed = bool(semantic_pass and subject_alignment and (matching_tokens if required else True)) + return { + "required": required, + "semantic_pass": semantic_pass, + "subject_alignment": subject_alignment, + "grounded_tool_semantic_hashes": grounded_tool_hashes, + "reply_receipt_tokens": reply_tokens, + "matching_receipt_tokens": matching_tokens, + "pass": passed, + } + + +def _score_semantic_results(results: list[dict[str, Any]], trial: dict[str, Any]) -> dict[str, Any]: + prompts = trial["prompts"] + regular_prompts = [item for item in prompts if item.get("custom_evidence_probe") is not True] + regular_ids = {item["id"] for item in regular_prompts} + regular_results = [item for item in results if str(item.get("prompt_id")) in regular_ids] + regular = benchmark.score_results( + regular_results, + memory_token=trial["memory_token"], + catalog=regular_prompts, + ) + regular_by_id = {item["prompt_id"]: item for item in regular["scores"]} + result_by_id = {str(item.get("prompt_id")): item for item in results if isinstance(item, dict)} + scores: list[dict[str, Any]] = [] + for prompt in prompts: + if prompt.get("custom_evidence_probe") is not True: + scores.append(regular_by_id.get(prompt["id"], {"prompt_id": prompt["id"], "pass": False})) + continue + reply = str((result_by_id.get(prompt["id"]) or {}).get("reply") or "") + checks = { + "reply_present": bool(reply.strip()), + "declares_read_only_mode": bool(re.search(r"\bmode\s*:\s*read-only\b", reply, re.I)), + "declares_context_surface": bool(re.search(r"\bsurface\s*:\s*context\b", reply, re.I)), + "no_write_completion_claim": not bool( + re.search(r"\b(?:i|we)\s+(?:applied|staged|wrote|updated|mutated)\b", reply, re.I) + ), + } + scores.append( + { + "prompt_id": prompt["id"], + "scorer_prompt_id": prompt["scorer_id"], + "dimension": prompt["dimension"], + "custom_signals": checks, + "pass": all(checks.values()), + } + ) + expected_ids = [item["id"] for item in prompts] + raw_ids = [str(item.get("prompt_id")) for item in results if isinstance(item, dict)] + missing = [prompt_id for prompt_id in expected_ids if prompt_id not in result_by_id] + unexpected = sorted(set(raw_ids) - set(expected_ids)) + duplicate_ids = sorted({prompt_id for prompt_id in raw_ids if raw_ids.count(prompt_id) > 1}) + return { + **regular, + "expected_prompt_ids": expected_ids, + "missing_prompt_ids": missing, + "unexpected_prompt_ids": unexpected, + "duplicate_prompt_ids": duplicate_ids, + "prompt_count": len(scores), + "passes": sum(1 for item in scores if item.get("pass") is True), + "failures": [item for item in scores if item.get("pass") is not True], + "scores": scores, + "pass": not missing + and not unexpected + and not duplicate_ids + and len(scores) == len(expected_ids) + and all(item.get("pass") is True for item in scores), + } + + +def _executed_behavior_ablation( + grounded_report: dict[str, Any], baseline_report: dict[str, Any] +) -> dict[str, Any]: + grounded = grounded_report.get("executed_behavior_manifest") or {} + baseline = baseline_report.get("executed_behavior_manifest") or {} + stable_keys = { + "schema", + "model_runtime", + "hermes_runtime", + "teleo_infrastructure_runtime", + "components", + "canonical_database", + } + + def stable(value: dict[str, Any]) -> dict[str, Any]: + return {key: value.get(key) for key in stable_keys} + + grounded_components = grounded.get("components") if isinstance(grounded.get("components"), dict) else {} + baseline_components = baseline.get("components") if isinstance(baseline.get("components"), dict) else {} + expected_component = "runtime_middleware" + grounded_middleware = grounded_components.get(expected_component) or {} + baseline_middleware = baseline_components.get(expected_component) or {} + grounded_content = grounded_middleware.get("content") or {} + baseline_content = baseline_middleware.get("content") or {} + grounded_files = { + str(item.get("path")): item + for item in grounded_content.get("files") or [] + if isinstance(item, dict) and item.get("path") + } + baseline_files = { + str(item.get("path")): item + for item in baseline_content.get("files") or [] + if isinstance(item, dict) and item.get("path") + } + extra_grounded_paths = set(grounded_files) - set(baseline_files) + extra_baseline_paths = set(baseline_files) - set(grounded_files) + common_paths = set(grounded_files) & set(baseline_files) + expected_db_context_path = "plugins/leo-db-context/__init__.py" + expected_db_context_manifest_path = "plugins/leo-db-context/plugin.yaml" + expected_removed_paths = {expected_db_context_path, expected_db_context_manifest_path} + instrumented_plugin_sha256 = hashlib.sha256( + instrument_db_context_plugin_source( + source_paths()["db_context_plugin_sha256"].read_text(encoding="utf-8") + ).encode("utf-8") + ).hexdigest() + checks = { + "manifest_hashes_valid": _valid_sha256(grounded.get("behavior_sha256")) + and grounded.get("behavior_sha256") == canonical_sha256(stable(grounded)) + and _valid_sha256(baseline.get("behavior_sha256")) + and baseline.get("behavior_sha256") == canonical_sha256(stable(baseline)), + "behavior_hashes_differ": grounded.get("behavior_sha256") != baseline.get("behavior_sha256"), + "component_sets_equal": bool(grounded_components) + and set(grounded_components) == set(baseline_components), + "non_middleware_components_equal": all( + grounded_components.get(name) == baseline_components.get(name) + for name in set(grounded_components) | set(baseline_components) + if name != expected_component + ), + "top_level_runtime_equal": all( + grounded.get(key) == baseline.get(key) + for key in stable_keys - {"components"} + ), + "middleware_metadata_equal": { + key: value for key, value in grounded_middleware.items() if key != "content" + } + == {key: value for key, value in baseline_middleware.items() if key != "content"}, + "middleware_nonfile_state_equal": grounded_content.get("missing") == baseline_content.get("missing") + and grounded_content.get("symlinks") == baseline_content.get("symlinks"), + "common_middleware_files_equal": bool(common_paths) + and all(grounded_files[path] == baseline_files[path] for path in common_paths), + "only_db_context_plugin_removed": extra_grounded_paths == expected_removed_paths + and not extra_baseline_paths, + "grounded_db_context_source_is_exact_instrumented_source": grounded_files.get( + expected_db_context_path, {} + ).get("sha256") + == instrumented_plugin_sha256, + "grounded_db_context_manifest_is_exact_frozen_source": grounded_files.get( + expected_db_context_manifest_path, {} + ).get("sha256") + == file_sha256(source_paths()["db_context_plugin_manifest_sha256"]), + } + return { + "expected_delta": "remove exact plugins/leo-db-context/{__init__.py,plugin.yaml}", + "extra_grounded_paths": sorted(extra_grounded_paths), + "extra_baseline_paths": sorted(extra_baseline_paths), + "instrumented_db_context_plugin_sha256": instrumented_plugin_sha256, + "checks": checks, + "pass": all(checks.values()), + } + + +def _receipt_score( + result: dict[str, Any], + *, + require_database_contract: bool, + require_database_receipt: bool, +) -> dict[str, Any]: + raw_traces = result.get("database_context_trace") or [] + traces = [item for item in raw_traces if isinstance(item, dict)] if isinstance(raw_traces, list) else [] + pre = [ + item + for item in traces + if item.get("event") == "pre_llm_call" and item.get("status") == "ok" and item.get("injected") is True + ] + post = [ + item + for item in traces + if item.get("event") == "post_llm_call" and item.get("status") == "ok" and item.get("validated") is True + ] + prompt_sha256 = hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest() + pre_hashes = {item.get("query_sha256") for item in pre if item.get("query_sha256")} + post_hashes = {item.get("query_sha256") for item in post if item.get("query_sha256")} + contract_ids_are_lists = all( + isinstance(item.get("contract_ids"), list) + and all(isinstance(contract_id, str) and contract_id for contract_id in item["contract_ids"]) + for item in pre + post + ) + contract_ids = { + str(contract_id) + for item in pre + post + for contract_id in (item.get("contract_ids") if isinstance(item.get("contract_ids"), list) else []) + if contract_id + } + raw_model_call_trace = result.get("model_call_trace") or [] + model_call_trace = ( + [item for item in raw_model_call_trace if isinstance(item, dict)] + if isinstance(raw_model_call_trace, list) + else [] + ) + retrieval_records = [] + for item in pre: + receipt = item.get("retrieval_receipt") if isinstance(item.get("retrieval_receipt"), dict) else {} + safe_receipt_payload = { + key: value + for key, value in receipt.items() + if key not in {"receipt_sha256", "trace_payload_sha256"} + } + trace_payload_sha256 = hashlib.sha256( + json.dumps(safe_receipt_payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + consistency = receipt.get("read_consistency") if isinstance(receipt.get("read_consistency"), dict) else {} + wal_before = consistency.get("wal_lsn_before") + wal_after = consistency.get("wal_lsn_after") + attempts = consistency.get("attempts") + typed_attempts = isinstance(attempts, int) and not isinstance(attempts, bool) and attempts >= 1 + consistency_evidence = bool( + wal_before + and wal_after + and typed_attempts + and ( + (consistency.get("status") == "stable_wal_marker" and wal_before == wal_after) + or ( + consistency.get("status") == "stable_content_across_wal_change_retry" + and attempts >= 2 + ) + ) + ) + if ( + item.get("source") == "kb_tool.py --local context" + and re.fullmatch(r"[0-9a-f]{64}", str(item.get("contract_sha256") or "")) + and item.get("compiled_response_available") is not None + and receipt.get("schema") == "livingip.teleoKbRetrievalReceipt.v1" + and receipt.get("query_sha256") == item.get("query_sha256") == prompt_sha256 + and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("semantic_context_sha256") or "")) + and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("artifact_state_sha256") or "")) + and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("receipt_sha256") or "")) + and receipt.get("trace_payload_sha256") == trace_payload_sha256 + and consistency.get("status") + in {"stable_wal_marker", "stable_content_across_wal_change_retry"} + and typed_attempts + and consistency.get("database") + and consistency.get("database_user") + and consistency.get("system_identifier") + and consistency_evidence + ): + retrieval_records.append(item) + supported_identifiers = { + str(identifier).lower() + for item in retrieval_records + for key in ("claim_ids", "source_ids") + for identifier in ((item.get("retrieval_receipt") or {}).get(key) or []) + } + reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(str(result.get("reply") or ""))} + unsupported_identifiers = sorted(reply_identifiers - supported_identifiers) + reply_sha256 = hashlib.sha256(str(result.get("reply") or "").encode()).hexdigest() + checks = { + "reply_present": result.get("ok") is True and bool(str(result.get("reply") or "").strip()), + "read_only_turn": result.get("mutates_kb") is False, + "trace_is_exact_typed_pair": isinstance(raw_traces, list) + and len(traces) == len(raw_traces) == 2 + and len(pre) == len(post) == 1, + "context_injected": len(pre) == 1, + "response_validated": len(post) == 1, + "contract_ids_are_typed_lists": contract_ids_are_lists, + "context_response_query_hash_bound": pre_hashes == post_hashes == {prompt_sha256}, + "delivered_response_hash_bound": len(post) == 1 + and post[0].get("delivered_response_sha256") == reply_sha256, + "database_contract_present": bool(contract_ids - NON_DB_CONTRACT_IDS) if require_database_contract else True, + "database_retrieval_receipt_present": bool(retrieval_records) if require_database_receipt else True, + "model_call_receipt_present": bool(model_call_trace) + and any(item.get("event") == "post_api_request" and item.get("model") and item.get("provider") for item in model_call_trace), + "conversation_history_prefix_preserved": result.get("conversation_history_prefix_preserved") is True, + "no_unsupported_exact_identifiers": not unsupported_identifiers, + } + return { + "checks": checks, + "contract_ids": sorted(contract_ids), + "query_sha256": sorted(pre_hashes & post_hashes), + "expected_prompt_sha256": prompt_sha256, + "database_tool_trace": result.get("database_tool_trace") or {}, + "reply_identifiers": sorted(reply_identifiers), + "supported_identifiers": sorted(supported_identifiers), + "unsupported_identifiers": unsupported_identifiers, + "pass": all(checks.values()), + } + + +def _benchmark_execution_chain(report: dict[str, Any]) -> dict[str, Any]: + """Validate the generic turn manifests under this benchmark's declared ablation. + + The generic manifest intentionally marks a dirty harness and missing DB-context + hooks incomplete. This benchmark permits exactly one control-owned dirty file + (``goal.md``) and, in the ablated arm only, the bindings made impossible by + removing the DB-context plugin. Every other runtime/model/session/safety + binding remains mandatory and is checked independently here. + """ + + mode = report.get("grounding_mode") + allowed_missing = ( + GROUNDED_EXECUTION_ALLOWED_MISSING + if mode == "grounded" + else ABLATION_EXECUTION_ALLOWED_MISSING + if mode == "db_tool_ablated" + else frozenset() + ) + results = [item for item in report.get("results") or [] if isinstance(item, dict)] + summary = report.get("execution_manifest_summary") or {} + executed_behavior = report.get("executed_behavior_manifest") or {} + local_state = report.get("oos_harness_git_state") or {} + summary_source = summary.get("harness_source") or {} + local_state_checks = { + "git_head_valid": _valid_git_revision(local_state.get("git_head")), + "status_sha256_valid": _valid_sha256(local_state.get("status_sha256")), + "recorded_dirty": local_state.get("worktree_clean") is False, + "only_control_goal_untracked": local_state.get("only_control_goal_untracked") is True + and local_state.get("status_lines") == ["?? goal.md"], + "generic_summary_source_bound": summary_source.get("git_head") == local_state.get("git_head") + and summary_source.get("status_sha256") == local_state.get("status_sha256") + and summary_source.get("worktree_clean") is False, + } + turn_checks: dict[str, dict[str, bool]] = {} + previous_execution_sha256: str | None = None + for index, result in enumerate(results): + prompt_id = str(result.get("prompt_id") or f"turn-{index + 1}") + manifest = result.get("execution_manifest") if isinstance(result.get("execution_manifest"), dict) else {} + turn = manifest.get("turn") or {} + runtime = manifest.get("runtime") or {} + model = manifest.get("model_execution") or {} + session = manifest.get("session_boundary") or {} + conversation = session.get("conversation") or {} + database = manifest.get("canonical_database") or {} + context = database.get("context_binding") or {} + tool_binding = database.get("database_tool_binding") or {} + delivery = manifest.get("delivery_and_safety") or {} + suite_safety = delivery.get("suite_safety") or {} + attribution = manifest.get("attribution") or {} + missing = attribution.get("missing_required_bindings") + missing_set = set(missing) if isinstance(missing, list) and all(isinstance(item, str) for item in missing) else set() + hermes_runtime = runtime.get("hermes_runtime") or {} + teleo_runtime = runtime.get("teleo_infrastructure_runtime") or {} + calls = model.get("calls") if isinstance(model.get("calls"), list) else [] + context_receipts = ( + database.get("context_retrieval_receipts") + if isinstance(database.get("context_retrieval_receipts"), list) + else [] + ) + checks = { + "generic_manifest_valid": bool(manifest) + and not execution_manifest_lib.validate_turn_manifest(manifest), + "prompt_bound": turn.get("prompt_id") == result.get("prompt_id") + and turn.get("prompt_sha256") == hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest(), + "reply_bound": turn.get("reply_sha256") + == hashlib.sha256(str(result.get("reply") or "").encode()).hexdigest(), + "declared_missing_exact": missing_set == allowed_missing + and attribution.get("status") == ("incomplete" if allowed_missing else "complete"), + "chain_bound": conversation.get("previous_execution_sha256") == previous_execution_sha256, + "session_bound": _valid_sha256(session.get("session_key_sha256")) + and session.get("source_platform") == "telegram" + and session.get("fresh_temp_profile_for_suite") is True + and session.get("prior_dynamic_state_excluded_from_suite") is True + and conversation.get("history_prefix_preserved") is True + and conversation.get("conversation_hashes_valid") is True + and conversation.get("prior_turn_state_bound") is True, + "runtime_bound": _valid_sha256(runtime.get("behavior_sha256")) + and runtime.get("behavior_sha256") == executed_behavior.get("behavior_sha256") + and _valid_git_revision(hermes_runtime.get("git_head")) + and _valid_sha256((hermes_runtime.get("source_tree") or {}).get("sha256")) + and hermes_runtime == executed_behavior.get("hermes_runtime") + and _valid_git_revision(teleo_runtime.get("git_head")) + and _valid_sha256((teleo_runtime.get("source_tree") or {}).get("sha256")) + and teleo_runtime == executed_behavior.get("teleo_infrastructure_runtime") + and runtime.get("harness_source") == summary_source, + "model_bound": isinstance(model.get("call_count"), int) + and not isinstance(model.get("call_count"), bool) + and model.get("call_count", 0) > 0 + and len(calls) == model.get("call_count") + and model.get("prompt_bound") is True + and model.get("delivered_response_bound") is True + and model.get("response_trace_count_matches_api_calls") is True + and model.get("api_call_sequence_valid") is True + and model.get("session_binding_valid") is True + and model.get("response_hashes_valid") is True + and (model.get("raw_response_bound") is (mode == "grounded")), + "database_state_bound": _valid_sha256((database.get("fingerprint_before") or {}).get("fingerprint_sha256")) + and (database.get("fingerprint_before") or {}).get("fingerprint_sha256") + == (database.get("fingerprint_after") or {}).get("fingerprint_sha256") + and database.get("fingerprint_unchanged") is True + and _valid_sha256(database.get("suite_counts_before_sha256")) + and database.get("suite_counts_before_sha256") == database.get("suite_counts_after_sha256") + and database.get("suite_counts_changed") is False, + "database_mode_bound": ( + len(context_receipts) == 1 + and database.get("binding_status") == "retrieval_receipt_bound" + and context.get("query_bound") is True + and context.get("context_available") is True + and context.get("response_bound") is True + ) + if mode == "grounded" + else ( + not context_receipts + and database.get("binding_status") == "missing" + and context.get("query_bound") is False + and context.get("context_available") is False + and context.get("response_bound") is False + ), + "database_tools_read_only": tool_binding.get("all_calls_read_only") is True, + "delivery_safe": delivery.get("posted_to_telegram") is False + and delivery.get("kb_mutation_by_harness") is False + and delivery.get("turn_mutates_kb") is False + and suite_safety.get("remote_returncode") == 0 + and suite_safety.get("pass_runtime") is True + and suite_safety.get("live_behavior_manifest_unchanged") is True + and suite_safety.get("temp_profile_removed") is True + and suite_safety.get("service_unchanged") is True + and suite_safety.get("db_fingerprint_unchanged") is True + and suite_safety.get("model_call_trace_all_bound") is True, + } + turn_checks[prompt_id] = checks + previous_execution_sha256 = manifest.get("execution_sha256") + checks = { + "recognized_grounding_mode": mode in {"grounded", "db_tool_ablated"}, + "results_nonempty": bool(results), + "summary_turn_count_exact": summary.get("turn_count") == len(results), + "one_manifest_per_result": bool(results) + and all(isinstance(item.get("execution_manifest"), dict) for item in results), + "local_harness_state_bound": all(local_state_checks.values()), + "all_turns_valid_under_declared_mode": bool(turn_checks) + and all(all(item.values()) for item in turn_checks.values()), + } + return { + "mode": mode, + "allowed_missing_bindings": sorted(allowed_missing), + "local_state_checks": local_state_checks, + "turn_checks": turn_checks, + "checks": checks, + "pass": all(checks.values()), + } + + +def _top_level_safety(report: dict[str, Any], *, require_handler_safety_gate: bool) -> dict[str, Any]: + before = report.get("db_fingerprint_before") or {} + after = report.get("db_fingerprint_after") or {} + service = report.get("service_before_after") or {} + benchmark_execution = _benchmark_execution_chain(report) + tool_surface = report.get("read_only_tool_surface") or {} + handler_safety = report.get("safety_gate") or {} + orphan_readback = report.get("post_run_orphan_readback") or {} + leakage_scan = report.get("prompt_leakage_scan") or {} + remote_leakage_scan = report.get("remote_temp_profile_prompt_leakage_scan") or {} + transport_deny = report.get("telegram_transport_deny") or {} + result_rows = [item for item in report.get("results") or [] if isinstance(item, dict)] + handler_failed = set(handler_safety.get("failed_checks") or []) + handler_checks = handler_safety.get("checks") if isinstance(handler_safety.get("checks"), dict) else {} + handler_gate_acceptable = handler_safety.get("status") == "pass" or bool( + handler_failed == {"all_turn_manifests_complete"} + and handler_checks + and all(value is True for key, value in handler_checks.items() if key != "all_turn_manifests_complete") + ) + checks = { + "fresh_temporary_session": (report.get("temp_profile_seed") or {}).get( + "same_session_continuity_starts_fresh" + ) + is True + and bool(result_rows) + and (result_rows[0].get("conversation_before") or {}).get("message_count") == 0, + "remote_returncode_zero": report.get("remote_returncode") == 0, + "runtime_passed": report.get("pass_runtime") is True, + "no_telegram_post": report.get("posted_to_telegram") is False, + "telegram_transport_deny_enabled": transport_deny.get("enabled") is True, + "zero_telegram_transport_attempts": isinstance(transport_deny.get("attempt_count"), int) + and not isinstance(transport_deny.get("attempt_count"), bool) + and transport_deny.get("attempt_count") == 0, + "telegram_send_method_patched": "send" in (transport_deny.get("patched_methods") or []), + "telegram_outbound_methods_exactly_denied": set(transport_deny.get("patched_methods") or []) + == EXPECTED_TELEGRAM_DENY_METHODS + and set(transport_deny.get("expected_methods") or []) == EXPECTED_TELEGRAM_DENY_METHODS, + "runner_adapters_empty": transport_deny.get("runner_adapters_empty") is True, + "harness_declared_no_kb_mutation": report.get("mutates_kb_by_harness") is False, + "database_counts_unchanged": report.get("db_counts_changed") is False, + "database_fingerprint_before_ok": before.get("status") == "ok", + "database_fingerprint_after_ok": after.get("status") == "ok", + "database_fingerprint_unchanged": report.get("db_fingerprint_unchanged") is True, + "database_fingerprint_hash_equal": bool( + before.get("fingerprint_sha256") + and before.get("fingerprint_sha256") == after.get("fingerprint_sha256") + ), + "live_behavior_manifest_unchanged": report.get("live_behavior_manifest_unchanged") is True, + "service_unchanged_during_trial": service.get("unchanged_from_preexisting_live_readback") is True, + "temporary_profile_removed": report.get("temp_profile_removed") is True, + "execution_chain_complete_under_declared_benchmark_mode": benchmark_execution["pass"], + "tool_registry_exactly_allowlisted": tool_surface.get("actual_registry_tools") + == ["skill_view", "skills_list", "terminal"], + "send_message_tool_absent": tool_surface.get("send_message_tool_enabled") is False, + "mutating_bridge_commands_not_exposed": tool_surface.get("mutating_bridge_commands_exposed") is False, + "terminal_provider_credentials_not_forwarded": tool_surface.get("provider_credentials_forwarded_to_terminal") + is False, + "terminal_restricted_to_exact_wrapper": tool_surface.get("terminal_restricted_to_exact_wrapper") is True, + "handler_safety_gate_passed_or_only_declared_manifest_gap": handler_gate_acceptable + if require_handler_safety_gate + else True, + "no_orphan_processes": orphan_readback.get("no_matching_processes") is True, + "prompt_leakage_scan_passed": leakage_scan.get("pass") is True, + "remote_temp_profile_prompt_leakage_scan_passed": remote_leakage_scan.get("pass") is True + and remote_leakage_scan.get("scope") + == "full_model_visible_temp_profile_excluding_sessions_state_memories_and_venv" + and isinstance(remote_leakage_scan.get("scanned_files"), int) + and not isinstance(remote_leakage_scan.get("scanned_files"), bool) + and remote_leakage_scan.get("scanned_files", 0) > 0 + and isinstance(remote_leakage_scan.get("scanned_bytes"), int) + and not isinstance(remote_leakage_scan.get("scanned_bytes"), bool) + and remote_leakage_scan.get("scanned_bytes", 0) > 0 + and remote_leakage_scan.get("errors") == [] + and { + item.get("name") + for item in remote_leakage_scan.get("expected_roots") or [] + if isinstance(item, dict) and item.get("exists") is True + } + == {"profile", "skills", "plugins", "bin"}, + } + return { + "checks": checks, + "benchmark_execution_chain": benchmark_execution, + "handler_safety_gate": { + "required": require_handler_safety_gate, + "acceptable": handler_gate_acceptable, + "failed_checks": sorted(handler_failed), + }, + "pass": all(checks.values()), + } + + +def ablate_receipts(report: dict[str, Any]) -> dict[str, Any]: + ablated = copy.deepcopy(report) + for result in ablated.get("results") or []: + result["database_context_trace"] = [] + result["database_tool_trace"] = {} + result["model_call_trace"] = [] + ablated["db_fingerprint_before"] = {"status": "ablated"} + ablated["db_fingerprint_after"] = {"status": "ablated"} + ablated["db_fingerprint_unchanged"] = False + ablated["turn_execution_manifests"] = [] + ablated["execution_manifest_summary"] = {"all_turns_attribution_complete": False} + return ablated + + +def _prompt_binding(report: dict[str, Any], trial: dict[str, Any]) -> dict[str, Any]: + expected = {item["id"]: item for item in trial["prompts"]} + raw_results = report.get("results") or [] + result_rows = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] + raw_ids = [str(item.get("prompt_id")) for item in result_rows] + actual = {str(item.get("prompt_id")): item for item in result_rows} + checks: dict[str, bool] = { + "results_are_objects": isinstance(raw_results, list) and len(result_rows) == len(raw_results), + "prompt_ids_exact": set(actual) == set(expected), + "prompt_count_exact": len(result_rows) == len(actual) == len(expected), + "prompt_ids_unique": len(raw_ids) == len(set(raw_ids)), + } + for prompt_id, prompt in expected.items(): + result = actual.get(prompt_id) or {} + checks[f"prompt_text:{prompt_id}"] = result.get("prompt") == prompt["message"] + checks[f"prompt_hash:{prompt_id}"] = hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest() == prompt[ + "message_sha256" + ] + return {"checks": checks, "pass": all(checks.values())} + + +def _valid_sha256(value: Any) -> bool: + return bool(re.fullmatch(r"[0-9a-f]{64}", str(value or ""))) + + +def _valid_git_revision(value: Any) -> bool: + return bool(re.fullmatch(r"[0-9a-f]{40}", str(value or ""))) + + +def _retained_path(value: Any) -> Path | None: + if not isinstance(value, str) or not value: + return None + path = Path(value) + if not path.is_absolute(): + path = Path(__file__).resolve().parents[1] / path + return path + + +def _nonempty_integer_mapping(value: Any) -> bool: + return bool( + isinstance(value, dict) + and value + and all(isinstance(key, str) and isinstance(item, int) and not isinstance(item, bool) for key, item in value.items()) + ) + + +def _parse_utc(value: Any) -> datetime | None: + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + return None + return parsed.astimezone(timezone.utc) + + +def _validate_restart_probe_reference( + reference: Any, + *, + report: dict[str, Any], +) -> dict[str, Any]: + reference = reference if isinstance(reference, dict) else {} + path = _retained_path(reference.get("path")) + payload: dict[str, Any] = {} + read_error: str | None = None + actual_sha256: str | None = None + if path is not None: + try: + raw = path.read_bytes() + actual_sha256 = hashlib.sha256(raw).hexdigest() + loaded = json.loads(raw) + if isinstance(loaded, dict): + payload = loaded + else: + read_error = "probe payload is not an object" + except (OSError, json.JSONDecodeError) as exc: + read_error = f"{type(exc).__name__}: {exc}" + before_counts = payload.get("db_counts_before") + after_counts = payload.get("db_counts_after") + before_fingerprint = payload.get("db_fingerprint_before") or {} + after_fingerprint = payload.get("db_fingerprint_after") or {} + transport = payload.get("telegram_transport_deny") or {} + results = payload.get("results") + checks = { + "reference_path_present": path is not None, + "reference_sha256_valid": _valid_sha256(reference.get("sha256")), + "artifact_loaded": read_error is None and bool(payload), + "artifact_sha256_matches": actual_sha256 is not None and actual_sha256 == reference.get("sha256"), + "protocol_id_bound": payload.get("protocol_id") == report.get("protocol_id"), + "protocol_hash_bound": payload.get("protocol_hash_sha256") == report.get("protocol_hash_sha256"), + "source_hashes_bound": payload.get("source_hashes") == report.get("source_hashes"), + "remote_runtime_passed": payload.get("remote_returncode") == 0 and payload.get("pass_runtime") is True, + "zero_prompt_probe": isinstance(results, list) and not results, + "no_telegram_post": payload.get("posted_to_telegram") is False, + "transport_deny_proven": transport.get("enabled") is True + and isinstance(transport.get("attempt_count"), int) + and not isinstance(transport.get("attempt_count"), bool) + and transport.get("attempt_count") == 0 + and transport.get("runner_adapters_empty") is True, + "transport_methods_exactly_denied": set(transport.get("patched_methods") or []) + == EXPECTED_TELEGRAM_DENY_METHODS + and set(transport.get("expected_methods") or []) == EXPECTED_TELEGRAM_DENY_METHODS, + "database_counts_complete_and_equal": _nonempty_integer_mapping(before_counts) + and before_counts == after_counts + and payload.get("db_counts_changed") is False, + "database_fingerprint_complete_and_equal": before_fingerprint.get("status") == "ok" + and after_fingerprint.get("status") == "ok" + and _valid_sha256(before_fingerprint.get("fingerprint_sha256")) + and before_fingerprint.get("fingerprint_sha256") == after_fingerprint.get("fingerprint_sha256") + and payload.get("db_fingerprint_unchanged") is True, + "preexecution_safety_passed": (payload.get("preexecution_safety_gate") or {}).get("status") == "pass", + "temporary_profile_removed": payload.get("temp_profile_removed") is True, + "no_orphan_processes": (payload.get("post_run_orphan_readback") or {}).get("no_matching_processes") is True, + } + return { + "path": reference.get("path") if isinstance(reference.get("path"), str) else None, + "actual_sha256": actual_sha256, + "read_error": read_error, + "checks": checks, + "pass": all(checks.values()), + "payload": payload, + } + + +def validate_restart_receipt(receipt: dict[str, Any] | None, report: dict[str, Any]) -> dict[str, Any]: + receipt = receipt if isinstance(receipt, dict) else {} + before = receipt.get("service_before") or {} + after = receipt.get("service_after") or {} + report_before = report.get("before_service") or {} + deploy_before = receipt.get("deploy_before") or {} + deploy_after = receipt.get("deploy_after") or {} + counts_before = receipt.get("db_counts_before") + counts_after = receipt.get("db_counts_after") + fingerprint_before = receipt.get("db_fingerprint_before") or {} + fingerprint_after = receipt.get("db_fingerprint_after") or {} + before_probe = _validate_restart_probe_reference(receipt.get("before_probe"), report=report) + after_probe = _validate_restart_probe_reference(receipt.get("after_probe"), report=report) + before_probe_payload = before_probe["payload"] + after_probe_payload = after_probe["payload"] + receipt_time = _parse_utc(receipt.get("generated_at_utc")) + report_time = _parse_utc(report.get("generated_at_utc")) + before_probe_time = _parse_utc(before_probe_payload.get("generated_at_utc")) + restart_started_time = _parse_utc(receipt.get("restart_started_at_utc")) + restart_ended_time = _parse_utc(receipt.get("restart_ended_at_utc")) + after_probe_time = _parse_utc(after_probe_payload.get("generated_at_utc")) + chronology_seconds = ( + (report_time - receipt_time).total_seconds() if receipt_time is not None and report_time is not None else None + ) + chronology_values = ( + before_probe_time, + restart_started_time, + restart_ended_time, + after_probe_time, + receipt_time, + report_time, + ) + deploy_revisions = [ + deploy_before.get("head"), + deploy_before.get("stamp"), + deploy_after.get("head"), + deploy_after.get("stamp"), + ] + self_checks = receipt.get("checks") if isinstance(receipt.get("checks"), dict) else {} + checks = { + "receipt_schema": receipt.get("schema") == "livingip.leoGatewayRestartReceipt.v1", + "protocol_id_bound": receipt.get("protocol_id") == report.get("protocol_id"), + "protocol_hash_bound": receipt.get("protocol_hash_sha256") == report.get("protocol_hash_sha256"), + "next_trial_bound": receipt.get("next_trial_id") == report.get("trial_id") + and receipt.get("next_trial_prompt_set_sha256") == report.get("trial_prompt_set_sha256"), + "chronology_bound": all(value is not None for value in chronology_values) + and list(chronology_values) == sorted(chronology_values) + and chronology_seconds is not None + and 0 <= chronology_seconds <= 3600, + "restart_command_succeeded": receipt.get("restart_returncode") == 0, + "service_active_after": after.get("ActiveState") == "active" and after.get("SubState") == "running", + "service_pid_changed": bool(before.get("MainPID") and before.get("MainPID") != after.get("MainPID")), + "trial_observed_restarted_pid": bool(after.get("MainPID") and report_before.get("MainPID") == after.get("MainPID")), + "service_start_identity_bound": bool( + after.get("ExecMainStartTimestamp") + and after.get("ExecMainStartTimestamp") == report_before.get("ExecMainStartTimestamp") + ), + "no_telegram_post": receipt.get("posted_to_telegram") is False, + "database_counts_complete_and_equal": _nonempty_integer_mapping(counts_before) + and counts_before == counts_after + and receipt.get("db_counts_changed") is False, + "database_fingerprint_complete_and_equal": fingerprint_before.get("status") == "ok" + and fingerprint_after.get("status") == "ok" + and _valid_sha256(fingerprint_before.get("fingerprint_sha256")) + and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256") + and receipt.get("db_fingerprint_unchanged") is True, + "deploy_identity_complete_and_equal": deploy_before.get("returncode") == 0 + and deploy_after.get("returncode") == 0 + and all(_valid_git_revision(value) for value in deploy_revisions) + and len(set(deploy_revisions)) == 1, + "before_probe_artifact_valid": before_probe["pass"], + "after_probe_artifact_valid": after_probe["pass"], + "probe_service_binding": (before_probe_payload.get("service_before_after") or {}).get("after") == before + and after_probe_payload.get("before_service") == after, + "probe_count_binding": before_probe_payload.get("db_counts_after") == counts_before + and after_probe_payload.get("db_counts_before") == counts_after, + "probe_fingerprint_binding": before_probe_payload.get("db_fingerprint_after") == fingerprint_before + and after_probe_payload.get("db_fingerprint_before") == fingerprint_after, + "receipt_self_checks_complete": bool(self_checks) and all(value is True for value in self_checks.values()), + "receipt_self_check_passed": receipt.get("pass") is True, + } + return { + "checks": checks, + "chronology_seconds": chronology_seconds, + "before_probe_validation": {key: value for key, value in before_probe.items() if key != "payload"}, + "after_probe_validation": {key: value for key, value in after_probe.items() if key != "payload"}, + "pass": all(checks.values()), + } + + +def score_live_trial( + protocol: dict[str, Any], + trial_id: str, + report: dict[str, Any], + *, + baseline_report: dict[str, Any], + restart_receipt: dict[str, Any] | None = None, +) -> dict[str, Any]: + protocol_validation = validate_protocol(protocol, verify_source_hashes=True) + trial = next((item for item in protocol.get("trials") or [] if item.get("trial_id") == trial_id), None) + if trial is None: + raise ValueError(f"unknown trial_id: {trial_id}") + prompt_binding = _prompt_binding(report, trial) + baseline_prompt_binding = _prompt_binding(baseline_report, trial) + report_results = [item for item in report.get("results") or [] if isinstance(item, dict)] + baseline_results = [item for item in baseline_report.get("results") or [] if isinstance(item, dict)] + semantic = _score_semantic_results(report_results, trial) + by_prompt = {item["id"]: item for item in trial["prompts"]} + receipts: dict[str, Any] = {} + subject_alignment: dict[str, bool] = {} + for result in report_results: + prompt_id = str(result.get("prompt_id") or "") + prompt = by_prompt.get(prompt_id) + if not prompt: + continue + receipts[prompt_id] = _receipt_score( + result, + require_database_contract=bool(prompt["requires_database_contract"]), + require_database_receipt=bool(prompt["requires_database_receipt"]), + ) + subject_alignment[prompt_id] = _subject_alignment(prompt, str(result.get("reply") or "")) + semantic_by_prompt = {item["prompt_id"]: item for item in semantic["scores"]} + report_by_prompt = {str(item.get("prompt_id")): item for item in report_results} + prompt_scores: list[dict[str, Any]] = [] + for prompt in trial["prompts"]: + prompt_id = prompt["id"] + semantic_item = semantic_by_prompt.get(prompt_id) or {"pass": False} + receipt_item = receipts.get(prompt_id) or {"pass": False, "checks": {}} + result_item = report_by_prompt.get(prompt_id) or {} + tool_evidence_hashes = _tool_evidence_hashes( + result_item, + expected_command_sha256=prompt.get("expected_tool_command_sha256"), + ) + evidence_answer = _evidence_answer_score( + prompt, + result_item, + semantic_pass=bool(semantic_item.get("pass")), + subject_alignment=bool(subject_alignment.get(prompt_id)), + grounded_tool_hashes=tool_evidence_hashes, + ) + grounded_pass = bool( + semantic_item.get("pass") and receipt_item.get("pass") and subject_alignment.get(prompt_id) + ) + prompt_scores.append( + { + "prompt_id": prompt_id, + "family_id": prompt["family_id"], + "scorer_id": prompt["scorer_id"], + "semantic_pass": bool(semantic_item.get("pass")), + "subject_alignment": bool(subject_alignment.get(prompt_id)), + "receipt_pass": bool(receipt_item.get("pass")), + "grounded_pass": grounded_pass, + "evidence_answer_pass": evidence_answer["pass"], + "evidence_answer_score": evidence_answer, + "semantic_score": semantic_item, + "receipt_score": receipt_item, + "reply_sha256": hashlib.sha256( + str((next((row for row in report_results if row.get("prompt_id") == prompt_id), {}) or {}).get("reply") or "").encode() + ).hexdigest(), + } + ) + top_safety = _top_level_safety(report, require_handler_safety_gate=True) + baseline_top_safety = _top_level_safety(baseline_report, require_handler_safety_gate=False) + restart_validation = ( + validate_restart_receipt(restart_receipt, report) + if trial["session_mode"] == "post_restart_clean_session" + else {"pass": True, "checks": {"not_a_restart_trial": True}} + ) + baseline_receipts = { + str(result.get("prompt_id")): _receipt_score( + result, + require_database_contract=bool(by_prompt.get(str(result.get("prompt_id")), {}).get("requires_database_contract")), + require_database_receipt=bool( + by_prompt.get(str(result.get("prompt_id")), {}).get("requires_database_receipt") + ), + ) + for result in baseline_results + if str(result.get("prompt_id")) in by_prompt + } + baseline_semantic = _score_semantic_results(baseline_results, trial) + grounded_passes = sum(1 for item in prompt_scores if item["grounded_pass"]) + prompt_count = len(prompt_scores) + baseline_semantic_by_prompt = {item["prompt_id"]: item for item in baseline_semantic["scores"]} + baseline_by_prompt = {str(item.get("prompt_id")): item for item in baseline_results} + baseline_subject_alignment = { + str(result.get("prompt_id")): _subject_alignment( + by_prompt[str(result.get("prompt_id"))], str(result.get("reply") or "") + ) + for result in baseline_results + if str(result.get("prompt_id")) in by_prompt + } + baseline_grounded_passes = sum( + 1 + for prompt in trial["prompts"] + if baseline_semantic_by_prompt.get(prompt["id"], {}).get("pass") + and baseline_subject_alignment.get(prompt["id"]) + and baseline_receipts.get(prompt["id"], {}).get("pass") + ) + evidence_prompt_ids = [ + prompt["id"] for prompt in trial["prompts"] if prompt.get("requires_tool_evidence_token") is True + ] + grounded_evidence_by_prompt = { + item["prompt_id"]: item["evidence_answer_score"]["grounded_tool_semantic_hashes"] + for item in prompt_scores + } + baseline_evidence_scores = { + prompt_id: _evidence_answer_score( + by_prompt[prompt_id], + baseline_by_prompt.get(prompt_id) or {}, + semantic_pass=bool(baseline_semantic_by_prompt.get(prompt_id, {}).get("pass")), + subject_alignment=bool(baseline_subject_alignment.get(prompt_id)), + grounded_tool_hashes=grounded_evidence_by_prompt.get(prompt_id) or [], + ) + for prompt_id in evidence_prompt_ids + } + current_evidence_passes = sum( + 1 + for item in prompt_scores + if item["prompt_id"] in evidence_prompt_ids and item["evidence_answer_pass"] is True + ) + baseline_evidence_passes = sum(1 for item in baseline_evidence_scores.values() if item["pass"] is True) + evidence_prompt_count = len(evidence_prompt_ids) + current_evidence_rate = current_evidence_passes / evidence_prompt_count if evidence_prompt_count else 0.0 + baseline_evidence_rate = baseline_evidence_passes / evidence_prompt_count if evidence_prompt_count else 0.0 + evidence_delta = current_evidence_rate - baseline_evidence_rate + baseline_no_db_checks = { + "grounding_mode": baseline_report.get("grounding_mode") == "db_tool_ablated", + "db_context_plugin_disabled": baseline_report.get("db_context_plugin_enabled") is False, + "tool_surface_ablation_mode": (baseline_report.get("read_only_tool_surface") or {}).get("mode") + == "no_db_ablation", + "zero_database_context_receipts": all( + not (item.get("database_context_trace") or []) for item in baseline_results + ), + "zero_successful_database_receipts": all(not item["pass"] for item in baseline_receipts.values()), + } + grounded_mode_checks = { + "grounding_mode": report.get("grounding_mode") == "grounded", + "db_context_plugin_enabled": report.get("db_context_plugin_enabled") is True, + "tool_surface_read_only_mode": (report.get("read_only_tool_surface") or {}).get("mode") == "read_only_kb", + "readonly_guard_bound_to_protocol": report.get("readonly_guard_source_sha256") + == protocol["source_hashes"]["readonly_guard_sha256"], + } + critical_prompt_checks = { + "all_receipt_gates_pass": all(item["receipt_pass"] for item in prompt_scores), + "no_unsupported_exact_identifiers": all( + not item["receipt_score"].get("unsupported_identifiers") for item in prompt_scores + ), + } + def model_identities(value: dict[str, Any]) -> list[dict[str, Any]]: + identities = { + canonical_sha256( + { + "model": item.get("model"), + "provider": item.get("provider"), + "base_url_sha256": item.get("base_url_sha256"), + "api_mode": item.get("api_mode"), + } + ): { + "model": item.get("model"), + "provider": item.get("provider"), + "base_url_sha256": item.get("base_url_sha256"), + "api_mode": item.get("api_mode"), + } + for result in value.get("results") or [] + if isinstance(result, dict) + for item in result.get("model_call_trace") or [] + if item.get("event") == "post_api_request" + } + return [identities[key] for key in sorted(identities)] + + executed_behavior_ablation = _executed_behavior_ablation(report, baseline_report) + comparison_axis_checks = { + "protocol_id_equal": baseline_report.get("protocol_id") == report.get("protocol_id") == protocol["protocol_id"], + "protocol_hash_equal": baseline_report.get("protocol_hash_sha256") + == report.get("protocol_hash_sha256") + == protocol["protocol_hash_sha256"], + "prompt_set_hash_equal": baseline_report.get("trial_prompt_set_sha256") + == report.get("trial_prompt_set_sha256") + == trial["prompt_set_sha256"], + "source_hashes_equal": baseline_report.get("source_hashes") == report.get("source_hashes") == protocol["source_hashes"], + "live_behavior_manifest_equal": bool( + (baseline_report.get("live_behavior_manifest_before") or {}).get("behavior_sha256") + and (baseline_report.get("live_behavior_manifest_before") or {}).get("behavior_sha256") + == (report.get("live_behavior_manifest_before") or {}).get("behavior_sha256") + ), + "executed_behavior_manifests_capture_only_declared_ablation": executed_behavior_ablation["pass"], + "temporary_profile_seed_equal": baseline_report.get("temp_profile_seed") == report.get("temp_profile_seed"), + "database_snapshot_equal_before_trials": (baseline_report.get("db_fingerprint_before") or {}).get( + "fingerprint_sha256" + ) + == (report.get("db_fingerprint_before") or {}).get("fingerprint_sha256"), + "model_provider_identity_equal": model_identities(baseline_report) == model_identities(report) + and bool(model_identities(report)), + "readonly_guard_source_equal_and_frozen": baseline_report.get("readonly_guard_source_sha256") + == report.get("readonly_guard_source_sha256") + == protocol["source_hashes"]["readonly_guard_sha256"], + "tool_schema_equal": (baseline_report.get("read_only_tool_surface") or {}).get("allowed_tools") + == (report.get("read_only_tool_surface") or {}).get("allowed_tools"), + } + threshold = float(protocol["thresholds"]["minimum_trial_grounded_pass_rate"]) + evidence_threshold = float(protocol["thresholds"]["minimum_trial_evidence_answer_pass_rate"]) + evidence_delta_threshold = float( + protocol["thresholds"]["minimum_current_minus_ablation_evidence_answer_delta"] + ) + score: dict[str, Any] = { + "schema": TRIAL_SCORE_SCHEMA, + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "protocol_id": protocol["protocol_id"], + "protocol_hash_sha256": protocol["protocol_hash_sha256"], + "source_hashes": protocol["source_hashes"], + "trial_id": trial_id, + "session_mode": trial["session_mode"], + "source_report_path": report.get("source_report_path"), + "protocol_validation": protocol_validation, + "prompt_binding": prompt_binding, + "baseline_prompt_binding": baseline_prompt_binding, + "top_level_safety": top_safety, + "baseline_top_level_safety": baseline_top_safety, + "grounded_mode_checks": grounded_mode_checks, + "critical_prompt_checks": critical_prompt_checks, + "baseline_no_db_checks": baseline_no_db_checks, + "comparison_axis_checks": comparison_axis_checks, + "executed_behavior_ablation": executed_behavior_ablation, + "restart_receipt_validation": restart_validation, + "semantic_score": semantic, + "prompt_scores": prompt_scores, + "grounded_passes": grounded_passes, + "prompt_count": prompt_count, + "grounded_pass_rate": grounded_passes / prompt_count if prompt_count else 0.0, + "evidence_answer_comparison": { + "method": "both_arms_scored_against_grounded_model_visible_tool_receipt_hashes", + "identical_replies_have_identical_evidence_answer_outcomes": True, + "prompt_ids": evidence_prompt_ids, + "prompt_count": evidence_prompt_count, + "current_passes": current_evidence_passes, + "current_pass_rate": current_evidence_rate, + "ablation_passes": baseline_evidence_passes, + "ablation_pass_rate": baseline_evidence_rate, + "current_minus_ablation_delta": evidence_delta, + "ablation_scores": baseline_evidence_scores, + }, + "receipt_ablation": { + "version": BASELINE_VERSION, + "same_prompt_set_sha256": trial["prompt_set_sha256"], + "semantic_passes": baseline_semantic["passes"], + "semantic_pass_rate": baseline_semantic["passes"] / prompt_count if prompt_count else 0.0, + "semantic_scores": baseline_semantic["scores"], + "grounded_passes": baseline_grounded_passes, + "grounded_pass_rate": baseline_grounded_passes / prompt_count if prompt_count else 0.0, + "receipt_scores": baseline_receipts, + "reply_sha256": { + str(item.get("prompt_id")): hashlib.sha256(str(item.get("reply") or "").encode()).hexdigest() + for item in baseline_results + }, + }, + "grounded_report_payload_sha256": canonical_sha256(report), + "baseline_report_payload_sha256": canonical_sha256(baseline_report), + } + score["pass"] = bool( + protocol_validation["pass"] + and prompt_binding["pass"] + and baseline_prompt_binding["pass"] + and top_safety["pass"] + and baseline_top_safety["pass"] + and all(grounded_mode_checks.values()) + and all(critical_prompt_checks.values()) + and all(baseline_no_db_checks.values()) + and all(comparison_axis_checks.values()) + and restart_validation["pass"] + and score["grounded_pass_rate"] >= threshold + and current_evidence_rate >= evidence_threshold + and evidence_delta >= evidence_delta_threshold + and score["receipt_ablation"]["grounded_passes"] == 0 + ) + score["derivation_core_sha256"] = canonical_sha256(score_derivation_core(score)) + return score + + +def validate_trial_score(protocol: dict[str, Any], trial: dict[str, Any], score: dict[str, Any]) -> dict[str, Any]: + def mapping(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + def number(value: Any) -> float: + try: + return float(value) + except (TypeError, ValueError): + return float("nan") + + def retained_report_checks(prefix: str) -> tuple[dict[str, bool], dict[str, Any]]: + path = _retained_path(score.get(f"{prefix}_report_path")) + payload: dict[str, Any] = {} + byte_sha256: str | None = None + if path is not None: + try: + raw = path.read_bytes() + byte_sha256 = hashlib.sha256(raw).hexdigest() + loaded = json.loads(raw) + if isinstance(loaded, dict): + payload = loaded + except (OSError, json.JSONDecodeError): + pass + return { + "path_present": path is not None, + "payload_loaded": bool(payload), + "byte_sha256_bound": _valid_sha256(score.get(f"{prefix}_report_sha256")) + and byte_sha256 == score.get(f"{prefix}_report_sha256"), + "canonical_payload_sha256_bound": bool(payload) + and canonical_sha256(payload) == score.get(f"{prefix}_report_payload_sha256"), + "protocol_id_bound": payload.get("protocol_id") == protocol.get("protocol_id"), + "protocol_hash_bound": payload.get("protocol_hash_sha256") == protocol.get("protocol_hash_sha256"), + "trial_id_bound": payload.get("trial_id") == trial.get("trial_id"), + "prompt_set_bound": payload.get("trial_prompt_set_sha256") == trial.get("prompt_set_sha256"), + "source_hashes_bound": payload.get("source_hashes") == protocol.get("source_hashes"), + }, payload + + def retained_restart_receipt_checks() -> tuple[dict[str, bool], dict[str, Any]]: + required = trial.get("session_mode") == "post_restart_clean_session" + path = _retained_path(score.get("restart_receipt_path")) + payload: dict[str, Any] = {} + byte_sha256: str | None = None + if path is not None: + try: + raw = path.read_bytes() + byte_sha256 = hashlib.sha256(raw).hexdigest() + loaded = json.loads(raw) + if isinstance(loaded, dict): + payload = loaded + except (OSError, json.JSONDecodeError): + pass + if not required: + return { + "not_required": True, + "path_absent": score.get("restart_receipt_path") is None, + "hashes_absent": score.get("restart_receipt_sha256") is None + and score.get("restart_receipt_payload_sha256") is None, + }, {} + return { + "required": True, + "path_present": path is not None, + "payload_loaded": bool(payload), + "byte_sha256_bound": _valid_sha256(score.get("restart_receipt_sha256")) + and byte_sha256 == score.get("restart_receipt_sha256"), + "canonical_payload_sha256_bound": bool(payload) + and canonical_sha256(payload) == score.get("restart_receipt_payload_sha256"), + "protocol_id_bound": payload.get("protocol_id") == protocol.get("protocol_id"), + "protocol_hash_bound": payload.get("protocol_hash_sha256") == protocol.get("protocol_hash_sha256"), + "next_trial_id_bound": payload.get("next_trial_id") == trial.get("trial_id"), + "next_prompt_set_bound": payload.get("next_trial_prompt_set_sha256") + == trial.get("prompt_set_sha256"), + }, payload + + expected_prompt_ids = [item["id"] for item in trial["prompts"]] + prompt_scores = score.get("prompt_scores") if isinstance(score.get("prompt_scores"), list) else [] + actual_prompt_ids = [str(item.get("prompt_id")) for item in prompt_scores if isinstance(item, dict)] + grounded_passes = sum(1 for item in prompt_scores if isinstance(item, dict) and item.get("grounded_pass") is True) + current_semantic_passes = sum( + 1 for item in prompt_scores if isinstance(item, dict) and item.get("semantic_pass") is True + ) + prompt_count = len(expected_prompt_ids) + baseline = mapping(score.get("receipt_ablation")) + baseline_receipts = baseline.get("receipt_scores") if isinstance(baseline.get("receipt_scores"), dict) else {} + baseline_semantic_scores = ( + baseline.get("semantic_scores") if isinstance(baseline.get("semantic_scores"), list) else [] + ) + baseline_semantic_ids = [ + str(item.get("prompt_id")) for item in baseline_semantic_scores if isinstance(item, dict) + ] + baseline_semantic_passes = sum( + 1 for item in baseline_semantic_scores if isinstance(item, dict) and item.get("pass") is True + ) + baseline_grounded_passes = int(baseline.get("grounded_passes") or 0) + evidence = mapping(score.get("evidence_answer_comparison")) + expected_evidence_ids = [ + item["id"] for item in trial["prompts"] if item.get("requires_tool_evidence_token") is True + ] + current_evidence_passes = sum( + 1 + for item in prompt_scores + if isinstance(item, dict) + and item.get("prompt_id") in expected_evidence_ids + and item.get("evidence_answer_pass") is True + ) + baseline_evidence_scores = mapping(evidence.get("ablation_scores")) + baseline_evidence_passes = sum( + 1 for item in baseline_evidence_scores.values() if isinstance(item, dict) and item.get("pass") is True + ) + evidence_count = len(expected_evidence_ids) + current_evidence_rate = current_evidence_passes / evidence_count if evidence_count else 0.0 + baseline_evidence_rate = baseline_evidence_passes / evidence_count if evidence_count else 0.0 + grounded_artifact_checks, grounded_payload = retained_report_checks("grounded") + baseline_artifact_checks, baseline_payload = retained_report_checks("baseline") + restart_artifact_checks, restart_payload = retained_restart_receipt_checks() + recomputed: dict[str, Any] = {} + if ( + grounded_payload + and baseline_payload + and (trial.get("session_mode") != "post_restart_clean_session" or restart_payload) + ): + try: + recomputed = score_live_trial( + protocol, + str(trial.get("trial_id") or ""), + grounded_payload, + baseline_report=baseline_payload, + restart_receipt=restart_payload or None, + ) + except (KeyError, TypeError, ValueError): + recomputed = {} + stored_core = score_derivation_core(score) + recomputed_core = score_derivation_core(recomputed) if recomputed else {} + checks = { + "schema": score.get("schema") == TRIAL_SCORE_SCHEMA, + "protocol_id": score.get("protocol_id") == protocol.get("protocol_id"), + "protocol_hash": score.get("protocol_hash_sha256") == protocol.get("protocol_hash_sha256"), + "source_hashes": score.get("source_hashes") == protocol.get("source_hashes"), + "trial_id": score.get("trial_id") == trial.get("trial_id"), + "session_mode": score.get("session_mode") == trial.get("session_mode"), + "prompt_ids_exact_and_ordered": actual_prompt_ids == expected_prompt_ids, + "prompt_count": score.get("prompt_count") == prompt_count == len(prompt_scores), + "grounded_passes_recomputed": score.get("grounded_passes") == grounded_passes, + "grounded_rate_recomputed": abs( + number(score.get("grounded_pass_rate")) - (grounded_passes / prompt_count if prompt_count else 0.0) + ) + < 1e-12, + "current_semantic_score_recomputed": mapping(score.get("semantic_score")).get("passes") + == current_semantic_passes, + "baseline_version": baseline.get("version") == protocol.get("baseline", {}).get("version"), + "baseline_receipt_ids_exact": set(baseline_receipts) == set(expected_prompt_ids), + "baseline_semantic_ids_exact_and_ordered": baseline_semantic_ids == expected_prompt_ids, + "baseline_semantic_passes_recomputed": baseline.get("semantic_passes") == baseline_semantic_passes, + "baseline_semantic_rate_recomputed": abs( + number(baseline.get("semantic_pass_rate")) + - (baseline_semantic_passes / prompt_count if prompt_count else 0.0) + ) + < 1e-12, + "baseline_grounded_rate_recomputed": abs( + number(baseline.get("grounded_pass_rate")) + - (baseline_grounded_passes / prompt_count if prompt_count else 0.0) + ) + < 1e-12, + "evidence_method_is_non_tautological": evidence.get("method") + == "both_arms_scored_against_grounded_model_visible_tool_receipt_hashes" + and evidence.get("identical_replies_have_identical_evidence_answer_outcomes") is True, + "evidence_prompt_ids_exact": evidence.get("prompt_ids") == expected_evidence_ids + and set(baseline_evidence_scores) == set(expected_evidence_ids), + "evidence_prompt_count": evidence.get("prompt_count") == evidence_count, + "current_evidence_passes_recomputed": evidence.get("current_passes") == current_evidence_passes, + "baseline_evidence_passes_recomputed": evidence.get("ablation_passes") == baseline_evidence_passes, + "current_evidence_rate_recomputed": abs( + number(evidence.get("current_pass_rate")) - current_evidence_rate + ) + < 1e-12, + "baseline_evidence_rate_recomputed": abs( + number(evidence.get("ablation_pass_rate")) - baseline_evidence_rate + ) + < 1e-12, + "evidence_delta_recomputed": abs( + number(evidence.get("current_minus_ablation_delta")) + - (current_evidence_rate - baseline_evidence_rate) + ) + < 1e-12, + "grounded_report_artifact_bound": bool(grounded_artifact_checks) + and all(grounded_artifact_checks.values()), + "baseline_report_artifact_bound": bool(baseline_artifact_checks) + and all(baseline_artifact_checks.values()), + "restart_receipt_artifact_bound": bool(restart_artifact_checks) + and all(restart_artifact_checks.values()), + "derivation_core_hash_bound": _valid_sha256(score.get("derivation_core_sha256")) + and score.get("derivation_core_sha256") == canonical_sha256(stored_core), + "score_recomputed_from_retained_artifacts": bool(recomputed) + and stored_core == recomputed_core + and score.get("derivation_core_sha256") == recomputed.get("derivation_core_sha256"), + "protocol_validation_passed": mapping(score.get("protocol_validation")).get("pass") is True, + "prompt_binding_passed": mapping(score.get("prompt_binding")).get("pass") is True, + "baseline_prompt_binding_passed": mapping(score.get("baseline_prompt_binding")).get("pass") is True, + "top_level_safety_passed": mapping(score.get("top_level_safety")).get("pass") is True, + "baseline_top_level_safety_passed": mapping(score.get("baseline_top_level_safety")).get("pass") is True, + "grounded_mode_checks_passed": bool(mapping(score.get("grounded_mode_checks"))) + and all(value is True for value in mapping(score.get("grounded_mode_checks")).values()), + "baseline_no_db_checks_passed": bool(mapping(score.get("baseline_no_db_checks"))) + and all(value is True for value in mapping(score.get("baseline_no_db_checks")).values()), + "comparison_axis_checks_passed": bool(mapping(score.get("comparison_axis_checks"))) + and all(value is True for value in mapping(score.get("comparison_axis_checks")).values()), + "critical_prompt_checks_passed": bool(mapping(score.get("critical_prompt_checks"))) + and all(value is True for value in mapping(score.get("critical_prompt_checks")).values()), + "restart_receipt_validation_passed": mapping(score.get("restart_receipt_validation")).get("pass") is True, + "score_passed": score.get("pass") is True, + } + return { + "pass": all(checks.values()), + "checks": checks, + "grounded_report_artifact_checks": grounded_artifact_checks, + "baseline_report_artifact_checks": baseline_artifact_checks, + "restart_receipt_artifact_checks": restart_artifact_checks, + "failed_checks": [k for k, v in checks.items() if not v], + } + + +def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str, Any]]) -> dict[str, Any]: + expected_ids = [item["trial_id"] for item in protocol.get("trials") or []] + by_id = {str(item.get("trial_id")): item for item in trial_scores} + trial_by_id = {item["trial_id"]: item for item in protocol.get("trials") or []} + integrity = { + trial_id: validate_trial_score(protocol, trial_by_id[trial_id], by_id.get(trial_id, {})) + for trial_id in expected_ids + } + current_rates = [float(by_id[trial_id].get("grounded_pass_rate") or 0.0) for trial_id in expected_ids if trial_id in by_id] + baseline_rates = [ + float((by_id[trial_id].get("receipt_ablation") or {}).get("grounded_pass_rate") or 0.0) + for trial_id in expected_ids + if trial_id in by_id + ] + evidence_current_rates = [ + float((by_id[trial_id].get("evidence_answer_comparison") or {}).get("current_pass_rate") or 0.0) + for trial_id in expected_ids + if trial_id in by_id + ] + evidence_baseline_rates = [ + float((by_id[trial_id].get("evidence_answer_comparison") or {}).get("ablation_pass_rate") or 0.0) + for trial_id in expected_ids + if trial_id in by_id + ] + semantic_current_rates = [ + float((by_id[trial_id].get("semantic_score") or {}).get("passes") or 0) + / max(1, int(by_id[trial_id].get("prompt_count") or 0)) + for trial_id in expected_ids + if trial_id in by_id + ] + semantic_baseline_rates = [ + float((by_id[trial_id].get("receipt_ablation") or {}).get("semantic_pass_rate") or 0.0) + for trial_id in expected_ids + if trial_id in by_id + ] + thresholds = protocol["thresholds"] + mean_current = statistics.fmean(current_rates) if current_rates else 0.0 + mean_baseline = statistics.fmean(baseline_rates) if baseline_rates else 0.0 + stddev = statistics.pstdev(current_rates) if len(current_rates) > 1 else 0.0 + mean_evidence_current = statistics.fmean(evidence_current_rates) if evidence_current_rates else 0.0 + mean_evidence_baseline = statistics.fmean(evidence_baseline_rates) if evidence_baseline_rates else 0.0 + evidence_stddev = statistics.pstdev(evidence_current_rates) if len(evidence_current_rates) > 1 else 0.0 + mean_semantic_current = statistics.fmean(semantic_current_rates) if semantic_current_rates else 0.0 + mean_semantic_baseline = statistics.fmean(semantic_baseline_rates) if semantic_baseline_rates else 0.0 + checks = { + "all_trials_present": len(trial_scores) == len(expected_ids) and set(by_id) == set(expected_ids), + "all_trial_scores_integrity_valid": all(item["pass"] for item in integrity.values()), + "all_trial_scores_pass": all(by_id.get(trial_id, {}).get("pass") is True for trial_id in expected_ids), + "minimum_trial_rate": bool(current_rates) + and min(current_rates) >= float(thresholds["minimum_trial_grounded_pass_rate"]), + "minimum_mean_rate": mean_current >= float(thresholds["minimum_mean_grounded_pass_rate"]), + "maximum_population_stddev": stddev <= float(thresholds["maximum_grounded_pass_rate_population_stddev"]), + "minimum_trial_evidence_answer_rate": bool(evidence_current_rates) + and min(evidence_current_rates) >= float(thresholds["minimum_trial_evidence_answer_pass_rate"]), + "minimum_mean_evidence_answer_rate": mean_evidence_current + >= float(thresholds["minimum_mean_evidence_answer_pass_rate"]), + "maximum_evidence_answer_population_stddev": evidence_stddev + <= float(thresholds["maximum_evidence_answer_pass_rate_population_stddev"]), + "minimum_non_tautological_evidence_ablation_delta": (mean_evidence_current - mean_evidence_baseline) + >= float(thresholds["minimum_current_minus_ablation_evidence_answer_delta"]), + "baseline_rejects_all_ungrounded_receipts": all(rate == 0.0 for rate in baseline_rates), + "broad_semantic_comparison_reported": len(semantic_current_rates) + == len(semantic_baseline_rates) + == len(expected_ids), + "restart_trial_passed": all( + by_id.get(item["trial_id"], {}).get("restart_receipt_validation", {}).get("pass") is True + for item in protocol["trials"] + if item["session_mode"] == "post_restart_clean_session" + ), + } + aggregate = { + "schema": AGGREGATE_SCHEMA, + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "protocol_id": protocol["protocol_id"], + "protocol_hash_sha256": protocol["protocol_hash_sha256"], + "scorer_version": protocol["scorer_version"], + "baseline_version": protocol["baseline"]["version"], + "trial_ids": expected_ids, + "current_grounded_pass_rates": current_rates, + "ablation_grounded_pass_rates": baseline_rates, + "mean_current_grounded_pass_rate": mean_current, + "mean_ablation_grounded_pass_rate": mean_baseline, + "current_minus_ablation_delta": mean_current - mean_baseline, + "current_grounded_pass_rate_population_stddev": stddev, + "minimum_current_grounded_pass_rate": min(current_rates) if current_rates else 0.0, + "current_evidence_answer_pass_rates": evidence_current_rates, + "ablation_evidence_answer_pass_rates": evidence_baseline_rates, + "mean_current_evidence_answer_pass_rate": mean_evidence_current, + "mean_ablation_evidence_answer_pass_rate": mean_evidence_baseline, + "current_minus_ablation_evidence_answer_delta": mean_evidence_current - mean_evidence_baseline, + "current_evidence_answer_pass_rate_population_stddev": evidence_stddev, + "minimum_current_evidence_answer_pass_rate": min(evidence_current_rates) + if evidence_current_rates + else 0.0, + "current_semantic_pass_rates": semantic_current_rates, + "ablation_semantic_pass_rates": semantic_baseline_rates, + "mean_current_semantic_pass_rate": mean_semantic_current, + "mean_ablation_semantic_pass_rate": mean_semantic_baseline, + "current_minus_ablation_semantic_delta": mean_semantic_current - mean_semantic_baseline, + "checks": checks, + "trial_score_integrity": integrity, + "trial_scores": trial_scores, + "required_tier": "T3_live_readonly", + "claim_ceiling": ( + "Repeated live VPS GatewayRunner reasoning with frozen blinded families and a current-build no-DB " + "ablation. Both arms are scored against the grounded arm's model-visible receipt tokens, so identical " + "answers cannot create the gated evidence delta. Broad semantic rates for both arms are reported " + "separately and are descriptive rather than silently inferred from missing receipts. Complete " + "tool/execution receipts and unchanged canonical database fingerprints are retained. No Telegram " + "delivery or production apply is proven." + ), + } + aggregate["pass"] = all(checks.values()) + return aggregate + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--freeze-protocol", type=Path) + mode.add_argument("--validate-protocol", type=Path) + mode.add_argument("--aggregate-protocol", type=Path) + parser.add_argument("--seed") + parser.add_argument("--trial-count", type=int, default=DEFAULT_TRIAL_COUNT) + parser.add_argument("--trial-score", type=Path, action="append", default=[]) + parser.add_argument("--out", type=Path) + args = parser.parse_args() + + if args.freeze_protocol: + if not args.seed: + raise SystemExit("--seed is required with --freeze-protocol") + protocol = freeze_protocol(args.seed, trial_count=args.trial_count) + args.freeze_protocol.parent.mkdir(parents=True, exist_ok=True) + args.freeze_protocol.write_text(json.dumps(protocol, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"protocol": str(args.freeze_protocol), "hash": protocol["protocol_hash_sha256"]}, indent=2)) + return 0 + + protocol_path = args.validate_protocol or args.aggregate_protocol + protocol = json.loads(protocol_path.read_text(encoding="utf-8")) + validation = validate_protocol(protocol, verify_source_hashes=True) + if args.validate_protocol: + print(json.dumps(validation, indent=2, sort_keys=True)) + return 0 if validation["pass"] else 1 + + scores = [json.loads(path.read_text(encoding="utf-8")) for path in args.trial_score] + aggregate = aggregate_trial_scores(protocol, scores) + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(aggregate, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(aggregate, indent=2, sort_keys=True)) + return 0 if aggregate["pass"] and validation["pass"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_leo_oos_readonly_guard.py b/tests/test_leo_oos_readonly_guard.py new file mode 100644 index 0000000..2933430 --- /dev/null +++ b/tests/test_leo_oos_readonly_guard.py @@ -0,0 +1,115 @@ +"""Fail-closed tests for the live OOS Hermes tool boundary.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import leo_oos_readonly_guard as guard # noqa: E402 + + +def make_wrapper(tmp_path: Path) -> tuple[Path, Path]: + profile = tmp_path / "profile" + wrapper = profile / "bin" / "teleo-kb" + wrapper.parent.mkdir(parents=True) + wrapper.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + wrapper.chmod(0o700) + return profile, wrapper + + +@pytest.mark.parametrize( + "command", + [ + "teleo-kb status --format json", + "teleo-kb search 'Orchid forecast' --limit 5 --context-limit 4 --format json", + "teleo-kb list-proposals --status approved --limit 10 --format markdown", + "teleo-kb show 11111111-1111-4111-8111-111111111111 --format json", + ], +) +def test_guard_accepts_only_exact_read_only_bridge_argv(tmp_path: Path, command: str) -> None: + profile, wrapper = make_wrapper(tmp_path) + argv, reason = guard.classify_terminal_command( + wrapper, + profile, + {"command": command}, + allow_kb_reads=True, + ) + assert argv is not None + assert reason is None + + +@pytest.mark.parametrize( + "tool_args", + [ + {"command": "teleo-kb propose-source /tmp/input"}, + {"command": "teleo-kb record-document-evaluation /tmp/input"}, + {"command": "teleo-kb status; rm -rf /tmp/x"}, + {"command": "bash -lc 'teleo-kb status'"}, + {"command": "teleo-kb status", "background": True}, + {"command": "teleo-kb status", "pty": True}, + {"command": "teleo-kb status", "workdir": "/tmp"}, + {"command": "teleo-kb evidence not-a-uuid"}, + {"command": "teleo-kb search x --limit 1000"}, + ], +) +def test_guard_rejects_writes_shell_escape_and_unbounded_arguments( + tmp_path: Path, tool_args: dict[str, object] +) -> None: + profile, wrapper = make_wrapper(tmp_path) + argv, reason = guard.classify_terminal_command( + wrapper, + profile, + tool_args, + allow_kb_reads=True, + ) + assert argv is None + assert reason + + +def test_no_db_ablation_preserves_terminal_schema_but_denies_execution(tmp_path: Path) -> None: + profile, wrapper = make_wrapper(tmp_path) + argv, reason = guard.classify_terminal_command( + wrapper, + profile, + {"command": "teleo-kb status --format json"}, + allow_kb_reads=False, + ) + assert argv is None + assert reason == "database tools are disabled for the no-DB ablation" + + +def test_terminal_child_uses_temp_profile_and_no_provider_credentials( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + profile, wrapper = make_wrapper(tmp_path) + captured: dict[str, object] = {} + + def fake_run(argv, **kwargs): + captured["argv"] = argv + captured["env"] = kwargs["env"] + return subprocess.CompletedProcess(argv, 0, stdout="{}\n", stderr="") + + monkeypatch.setattr(guard.subprocess, "run", fake_run) + output = json.loads( + guard.restricted_bridge_terminal_handler( + wrapper, + profile, + {"command": "teleo-kb status --format json"}, + allow_kb_reads=True, + ) + ) + assert output["exit_code"] == 0 + assert captured["env"] == { + "HOME": str(profile), + "HERMES_HOME": str(profile), + "PATH": f"{profile / 'bin'}:{guard.os.environ.get('PATH', '')}", + "TELEO_KB_MODE": "local", + } + assert not any("KEY" in key or "TOKEN" in key or "SECRET" in key for key in captured["env"]) diff --git a/tests/test_run_leo_m3taversal_oos_handler_suite.py b/tests/test_run_leo_m3taversal_oos_handler_suite.py new file mode 100644 index 0000000..a485ae9 --- /dev/null +++ b/tests/test_run_leo_m3taversal_oos_handler_suite.py @@ -0,0 +1,117 @@ +"""Static and safety tests for the guarded live OOS runner.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import run_leo_m3taversal_oos_handler_suite as suite # noqa: E402 +import working_leo_m3taversal_oos_protocol as protocol_lib # noqa: E402 + + +def protocol() -> dict: + return protocol_lib.freeze_protocol( + "runner-test-seed", + created_at_utc="2026-07-15T00:00:00+00:00", + ) + + +@pytest.mark.parametrize("grounding_mode", suite.GROUNDING_MODES) +def test_guarded_remote_script_compiles_and_installs_preexecution_gate(grounding_mode: str) -> None: + script = suite.build_guarded_remote_script( + "cafef00d", + prompts=[{"id": "P1", "dimension": "demo", "message": "Read the database only"}], + suite_mode="test-mode", + report_prefix="oos-test-report", + prompt_note="frozen test", + grounding_mode=grounding_mode, + leakage_scan={"pass": True, "exact_prompt_matches": []}, + ) + compile(script, "", "exec") + assert "install_read_only_tool_surface" in script + assert "trace_payload_sha256" in script + assert 'report["preexecution_safety_gate"]' in script + assert 'report["remote_temp_profile_prompt_leakage_scan"]' in script + assert '"errors": remote_scan_errors' in script + assert "scan_path.resolve(strict=True)" in script + assert "2_000_000" not in script + assert 'raise RuntimeError("OOS pre-execution safety gate failed")' in script + assert "send_message_tool_enabled" in script + assert '"scope": "full_model_visible_temp_profile_excluding_sessions_state_memories_and_venv"' in script + assert "Telegram transport is denied in the OOS no-post harness" in script + assert "runner.adapters.clear()" in script + assert 'report["executed_behavior_manifest"] = build_behavior_manifest(temp_profile)' in script + assert script.index('report["executed_behavior_manifest"] = build_behavior_manifest(temp_profile)') < script.index( + "source = SessionSource(" + ) + assert "timeout=180" in script + assert "timeout=300" not in script + if grounding_mode == "db_tool_ablated": + assert 'shutil.rmtree(db_context_dir)' in script + + +def test_prompt_leakage_scan_uses_partial_markers_and_runtime_roots( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + frozen = protocol() + marker = frozen["trials"][0]["prompts"][0]["leakage_markers"][1] + runtime_root = tmp_path / "skills" + runtime_root.mkdir() + (runtime_root / "leaked.md").write_text(f"prefix {marker} suffix", encoding="utf-8") + monkeypatch.setattr(suite, "RUNTIME_PROMPT_SCAN_ROOTS", (runtime_root,)) + scan = suite.prompt_leakage_scan(frozen) + assert scan["pass"] is False + assert scan["exact_prompt_matches"][0]["prompt_id"] == frozen["trials"][0]["prompts"][0]["id"] + assert scan["exact_prompt_matches"][0]["marker_sha256"] + + +def test_current_repo_runtime_has_no_frozen_prompt_marker_leakage() -> None: + scan = suite.prompt_leakage_scan(protocol()) + assert scan["pass"] is True, scan["exact_prompt_matches"] + assert scan["scanned_files"] > 0 + + +def test_restart_probe_requires_full_fingerprint_guard_and_cleanup() -> None: + counts = {"public.claims": 2} + fingerprint = {"status": "ok", "fingerprint_sha256": "a" * 64} + report = { + "remote_returncode": 0, + "pass_runtime": True, + "posted_to_telegram": False, + "db_counts_changed": False, + "db_counts_before": counts, + "db_counts_after": counts, + "db_fingerprint_unchanged": True, + "db_fingerprint_before": fingerprint, + "db_fingerprint_after": fingerprint, + "preexecution_safety_gate": {"status": "pass"}, + "telegram_transport_deny": { + "enabled": True, + "attempt_count": 0, + "runner_adapters_empty": True, + }, + "temp_profile_removed": True, + "post_run_orphan_readback": {"no_matching_processes": True}, + } + assert suite._restart_probe_passes(report) is True + for path in ( + ("db_fingerprint_unchanged",), + ("preexecution_safety_gate", "status"), + ("post_run_orphan_readback", "no_matching_processes"), + ("telegram_transport_deny", "enabled"), + ): + broken = {**report} + if len(path) == 1: + broken[path[0]] = False + else: + broken[path[0]] = {**report[path[0]], path[1]: False} + assert suite._restart_probe_passes(broken) is False + + +def test_portable_artifact_paths_are_repo_relative() -> None: + assert suite._portable_artifact_path(ROOT / "pyproject.toml") == "pyproject.toml" diff --git a/tests/test_working_leo_m3taversal_oos_protocol.py b/tests/test_working_leo_m3taversal_oos_protocol.py new file mode 100644 index 0000000..9959ac5 --- /dev/null +++ b/tests/test_working_leo_m3taversal_oos_protocol.py @@ -0,0 +1,975 @@ +"""Tests for the frozen blinded protocol, receipt gates, and aggregation.""" + +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import working_leo_m3taversal_oos_protocol as protocol_lib # noqa: E402 + + +def load_legacy_test_helpers(): + path = ROOT / "tests" / "test_working_leo_m3taversal_oos_benchmark.py" + spec = importlib.util.spec_from_file_location("oos_legacy_test_helpers", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +LEGACY = load_legacy_test_helpers() + + +def frozen_protocol() -> dict: + return protocol_lib.freeze_protocol( + "unit-test-seed-without-live-output", + created_at_utc="2026-07-15T00:00:00+00:00", + ) + + +def tool_surface(mode: str) -> dict: + return { + "mode": mode, + "allowed_tools": ["skills_list", "skill_view", "terminal"], + "actual_registry_tools": ["skill_view", "skills_list", "terminal"], + "read_only_bridge_commands": ["status"] if mode == "read_only_kb" else [], + "send_message_tool_enabled": False, + "terminal_restricted_to_exact_wrapper": True, + "mutating_bridge_commands_exposed": False, + "provider_credentials_forwarded_to_terminal": False, + } + + +def retrieval_receipt(query_sha256: str) -> dict: + receipt = { + "schema": "livingip.teleoKbRetrievalReceipt.v1", + "query_sha256": query_sha256, + "semantic_context_sha256": "1" * 64, + "artifact_state_sha256": "2" * 64, + "receipt_sha256": "3" * 64, + "claim_ids": [], + "source_ids": [], + "counts": {"claims": 10}, + "read_consistency": { + "status": "stable_wal_marker", + "attempts": 1, + "database": "teleo", + "database_user": "postgres", + "system_identifier": "12345", + "wal_lsn_before": "0/1", + "wal_lsn_after": "0/1", + }, + } + trace_payload = {key: value for key, value in receipt.items() if key != "receipt_sha256"} + receipt["trace_payload_sha256"] = hashlib.sha256( + json.dumps(trace_payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + return receipt + + +def result_for(prompt: dict, token: str, turn: int, *, grounded: bool) -> dict: + query_sha256 = hashlib.sha256(prompt["message"].encode()).hexdigest() + if prompt.get("custom_evidence_probe"): + reply = f"Subject: {prompt['subject']}\nMode: read-only\nSurface: context" + else: + reply = f"{prompt['subject']}. {LEGACY.good_reply(prompt['scorer_id'], token)}" + if grounded and prompt["requires_tool_evidence_token"]: + reply += "\nReceipt: aaaaaaaaaaaa" + contexts = [] + if grounded: + contexts = [ + { + "event": "pre_llm_call", + "status": "ok", + "injected": True, + "source": "kb_tool.py --local context", + "query_sha256": query_sha256, + "contract_ids": ["reply_budget", "demo_capability_readback"], + "contract_sha256": "4" * 64, + "compiled_response_available": True, + "retrieval_receipt": retrieval_receipt(query_sha256), + }, + { + "event": "post_llm_call", + "status": "ok", + "validated": True, + "query_sha256": query_sha256, + "contract_ids": ["reply_budget", "demo_capability_readback"], + "delivered_response_sha256": hashlib.sha256(reply.encode()).hexdigest(), + }, + ] + tool_calls = [] + if grounded and prompt["requires_tool_evidence_token"]: + tool_calls = [ + { + "database_invocations": [ + { + "access_mode": "read_only", + "executable": "teleo-kb", + "subcommand": "context", + "command_sha256": prompt["expected_tool_command_sha256"], + } + ], + "result": { + "nonempty": True, + "error_detected": False, + "retrieval_receipt": { + "schema": "livingip.teleoKbRetrievalReceipt.v1", + "semantic_context_sha256": "a" * 64, + "artifact_state_sha256": "b" * 64, + "read_consistency_status": "stable_wal_marker", + }, + }, + } + ] + return { + "turn": turn, + "prompt_id": prompt["id"], + "dimension": prompt["dimension"], + "prompt": prompt["message"], + "reply": reply, + "ok": True, + "mutates_kb": False, + "database_context_trace": contexts, + "database_tool_trace": { + "schema": "livingip.leoKbToolTrace.v1", + "database_tool_call_count": len(tool_calls), + "database_tool_completed_count": len(tool_calls), + "database_retrieval_receipt_proven": bool(tool_calls), + "database_tool_calls_read_only": bool(tool_calls), + "access_modes": ["read_only"] if tool_calls else [], + "calls": tool_calls, + }, + "model_call_trace": [ + { + "event": "post_api_request", + "model": "test-model", + "provider": "test-provider", + "base_url_sha256": "5" * 64, + "api_mode": "chat", + "response_model": "test-model", + } + ], + "conversation_before": {"message_count": 0 if turn == 1 else (turn - 1) * 2}, + "conversation_after": {"message_count": turn * 2}, + "conversation_history_prefix_preserved": True, + } + + +def fake_behavior_manifest(*, grounded: bool) -> dict: + def component(name: str, files: list[dict] | None = None) -> dict: + file_rows = files or [ + { + "path": f"{name}.txt", + "bytes": 1, + "mode": "0o644", + "sha256": hashlib.sha256(name.encode()).hexdigest(), + } + ] + content_stable = {"files": file_rows, "missing": [], "symlinks": []} + return { + "role": f"test {name}", + "mutability": "test_static", + "replayability": "fully_hashable", + "content": { + **content_stable, + "file_count": len(file_rows), + "total_bytes": sum(int(item["bytes"]) for item in file_rows), + "sha256": protocol_lib.canonical_sha256(content_stable), + }, + } + + common_middleware = { + "path": "plugins/leo-turn-trace/__init__.py", + "bytes": 1, + "mode": "0o644", + "sha256": "3" * 64, + } + middleware_files = [common_middleware] + if grounded: + instrumented = protocol_lib.instrument_db_context_plugin_source( + protocol_lib.source_paths()["db_context_plugin_sha256"].read_text(encoding="utf-8") + ) + middleware_files.append( + { + "path": "plugins/leo-db-context/__init__.py", + "bytes": len(instrumented.encode()), + "mode": "0o644", + "sha256": hashlib.sha256(instrumented.encode()).hexdigest(), + } + ) + plugin_manifest_path = protocol_lib.source_paths()["db_context_plugin_manifest_sha256"] + middleware_files.append( + { + "path": "plugins/leo-db-context/plugin.yaml", + "bytes": plugin_manifest_path.stat().st_size, + "mode": "0o644", + "sha256": hashlib.sha256(plugin_manifest_path.read_bytes()).hexdigest(), + } + ) + components = { + name: component(name) + for name in ( + "profile_config", + "runtime_identity", + "procedural_skills", + "database_tools", + "persistent_memory", + "conversation_sessions", + "generated_prompt_cache", + ) + } + components["runtime_middleware"] = component("runtime_middleware", middleware_files) + stable = { + "schema": "livingip.leoBehaviorManifest.v1", + "model_runtime": {"model": "test-model"}, + "hermes_runtime": { + "git_head": "e" * 40, + "source_tree": {"sha256": "f" * 64, "file_count": 1}, + }, + "teleo_infrastructure_runtime": { + "git_head": "1" * 40, + "source_tree": {"sha256": "2" * 64, "file_count": 1}, + }, + "components": components, + "canonical_database": {"included": False}, + } + return { + **stable, + "generated_at_utc": "2026-07-15T00:01:00+00:00", + "profile_root": "/tmp/profile", + "hermes_root": "/opt/hermes", + "deployment_root": "/opt/teleo", + "behavior_sha256": protocol_lib.canonical_sha256(stable), + } + + +def attach_fake_execution_manifests(report: dict, *, grounded: bool) -> None: + harness_source = { + "git_head": "a" * 40, + "worktree_clean": False, + "status_sha256": hashlib.sha256(b"?? goal.md\n").hexdigest(), + } + previous_execution_sha256 = None + allowed_missing = ( + protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING + if grounded + else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING + ) + fingerprint = report["db_fingerprint_before"] + counts_sha256 = protocol_lib.canonical_sha256({}) + for result in report["results"]: + conversation = { + "history_prefix_preserved": True, + "conversation_hashes_valid": True, + "prior_turn_state_bound": True, + "previous_execution_sha256": previous_execution_sha256, + } + stable = { + "schema": protocol_lib.execution_manifest_lib.SCHEMA, + "turn": { + "prompt_id": result["prompt_id"], + "prompt_sha256": hashlib.sha256(result["prompt"].encode()).hexdigest(), + "reply_sha256": hashlib.sha256(result["reply"].encode()).hexdigest(), + }, + "session_boundary": { + "session_key_sha256": "b" * 64, + "source_platform": "telegram", + "fresh_temp_profile_for_suite": True, + "prior_dynamic_state_excluded_from_suite": True, + "conversation": conversation, + }, + "runtime": { + "behavior_sha256": report["executed_behavior_manifest"]["behavior_sha256"], + "hermes_runtime": report["executed_behavior_manifest"]["hermes_runtime"], + "teleo_infrastructure_runtime": report["executed_behavior_manifest"][ + "teleo_infrastructure_runtime" + ], + "harness_source": harness_source, + }, + "model_execution": { + "call_count": 1, + "calls": [{"model": "test-model", "provider": "test-provider"}], + "prompt_bound": True, + "raw_response_bound": grounded, + "delivered_response_bound": True, + "response_trace_count_matches_api_calls": True, + "api_call_sequence_valid": True, + "session_binding_valid": True, + "response_hashes_valid": True, + }, + "canonical_database": { + "binding_status": "retrieval_receipt_bound" if grounded else "missing", + "context_retrieval_receipts": [retrieval_receipt(hashlib.sha256(result["prompt"].encode()).hexdigest())] + if grounded + else [], + "context_binding": { + "query_bound": grounded, + "context_available": grounded, + "response_bound": grounded, + }, + "database_tool_binding": {"all_calls_read_only": True}, + "fingerprint_before": fingerprint, + "fingerprint_after": fingerprint, + "fingerprint_unchanged": True, + "suite_counts_before_sha256": counts_sha256, + "suite_counts_after_sha256": counts_sha256, + "suite_counts_changed": False, + }, + "delivery_and_safety": { + "posted_to_telegram": False, + "kb_mutation_by_harness": False, + "turn_mutates_kb": False, + "suite_safety": { + "remote_returncode": 0, + "pass_runtime": True, + "live_behavior_manifest_unchanged": True, + "temp_profile_removed": True, + "service_unchanged": True, + "db_fingerprint_unchanged": True, + "model_call_trace_all_bound": True, + }, + }, + "attribution": { + "status": "incomplete", + "missing_required_bindings": sorted(allowed_missing), + }, + } + manifest = { + **stable, + "generated_at_utc": "2026-07-15T00:02:00+00:00", + "execution_sha256": protocol_lib.execution_manifest_lib.canonical_sha256(stable), + } + result["execution_manifest"] = manifest + previous_execution_sha256 = manifest["execution_sha256"] + report["oos_harness_git_state"] = { + **harness_source, + "status_lines": ["?? goal.md"], + "only_control_goal_untracked": True, + } + report["execution_manifest_summary"] = { + "schema": protocol_lib.execution_manifest_lib.SCHEMA, + "turn_count": len(report["results"]), + "attribution_complete_count": 0, + "all_turns_attribution_complete": False, + "harness_source": harness_source, + } + + +def fake_report(protocol: dict, trial: dict, *, grounded: bool) -> dict: + mode = "grounded" if grounded else "db_tool_ablated" + fingerprint = { + "status": "ok", + "fingerprint_sha256": "6" * 64, + "table_rows_sha256": "7" * 64, + "structure_sha256": "8" * 64, + } + results = [ + result_for(prompt, trial["memory_token"], index, grounded=grounded) + for index, prompt in enumerate(trial["prompts"], start=1) + ] + report = { + "protocol_id": protocol["protocol_id"], + "protocol_hash_sha256": protocol["protocol_hash_sha256"], + "trial_id": trial["trial_id"], + "trial_prompt_set_sha256": trial["prompt_set_sha256"], + "source_hashes": protocol["source_hashes"], + "source_report_path": f"/tmp/{trial['trial_id']}-{mode}.json", + "generated_at_utc": "2026-07-15T00:02:00+00:00", + "grounding_mode": mode, + "db_context_plugin_enabled": grounded, + "remote_returncode": 0, + "pass_runtime": True, + "posted_to_telegram": False, + "mutates_kb_by_harness": False, + "db_counts_changed": False, + "db_fingerprint_before": fingerprint, + "db_fingerprint_after": fingerprint, + "db_fingerprint_unchanged": True, + "live_behavior_manifest_before": {"behavior_sha256": "9" * 64}, + "live_behavior_manifest_after": {"behavior_sha256": "9" * 64}, + "live_behavior_manifest_unchanged": True, + "service_before_after": { + "before": {"MainPID": "202", "ActiveState": "active", "SubState": "running"}, + "after": {"MainPID": "202", "ActiveState": "active", "SubState": "running"}, + "unchanged_from_preexisting_live_readback": True, + }, + "before_service": { + "MainPID": "202", + "ActiveState": "active", + "SubState": "running", + "ExecMainStartTimestamp": "Wed 2026-07-15 00:01:00 UTC", + }, + "temp_profile_removed": True, + "temp_profile_seed": { + "mode": "static_runtime_surfaces_only", + "excluded": ["sessions", "state", "state.db"], + "same_session_continuity_starts_fresh": True, + }, + "executed_behavior_manifest": fake_behavior_manifest(grounded=grounded), + "read_only_tool_surface": tool_surface("read_only_kb" if grounded else "no_db_ablation"), + "readonly_guard_source_sha256": protocol["source_hashes"]["readonly_guard_sha256"], + "safety_gate": { + "status": "fail", + "checks": { + "remote_command_succeeded": True, + "runtime_completed": True, + "handler_authorized": True, + "results_nonempty": True, + "all_turns_returned_replies": True, + "all_turns_declared_no_mutation": True, + "all_tool_traces_read_only_and_bound": True, + "no_telegram_post": True, + "harness_declared_no_kb_mutation": True, + "database_counts_unchanged": True, + "database_fingerprint_unchanged": True, + "live_behavior_unchanged": True, + "service_unchanged": True, + "temporary_profile_removed": True, + "model_trace_metadata_bound": True, + "all_turn_manifests_complete": False, + "no_runtime_error": True, + }, + "failed_checks": ["all_turn_manifests_complete"], + }, + "post_run_orphan_readback": {"no_matching_processes": True}, + "prompt_leakage_scan": {"pass": True}, + "remote_temp_profile_prompt_leakage_scan": { + "scope": "full_model_visible_temp_profile_excluding_sessions_state_memories_and_venv", + "expected_roots": [ + {"name": name, "exists": True, "is_symlink": False} + for name in ("profile", "skills", "plugins", "bin") + ], + "scanned_files": 10, + "scanned_bytes": 1000, + "errors": [], + "matches": [], + "pass": True, + }, + "telegram_transport_deny": { + "enabled": True, + "patched_methods": sorted(protocol_lib.EXPECTED_TELEGRAM_DENY_METHODS), + "expected_methods": sorted(protocol_lib.EXPECTED_TELEGRAM_DENY_METHODS), + "attempt_count": 0, + "runner_adapters_empty": True, + }, + "results": results, + } + attach_fake_execution_manifests(report, grounded=grounded) + return report + + +def restart_receipt(protocol: dict, directory: Path, trial: dict) -> dict: + fingerprint = {"status": "ok", "fingerprint_sha256": "6" * 64} + counts = {"public.claims": 10, "public.sources": 2} + deploy = {"returncode": 0, "head": "a" * 40, "stamp": "a" * 40} + service_before = { + "MainPID": "101", + "ActiveState": "active", + "SubState": "running", + "ExecMainStartTimestamp": "Tue 2026-07-14 23:00:00 UTC", + } + service_after = { + "MainPID": "202", + "ActiveState": "active", + "SubState": "running", + "ExecMainStartTimestamp": "Wed 2026-07-15 00:01:00 UTC", + } + + def probe(path: Path, *, before_restart: bool) -> dict: + payload = { + "generated_at_utc": "2026-07-15T00:00:00+00:00" + if before_restart + else "2026-07-15T00:01:15+00:00", + "protocol_id": protocol["protocol_id"], + "protocol_hash_sha256": protocol["protocol_hash_sha256"], + "source_hashes": protocol["source_hashes"], + "remote_returncode": 0, + "pass_runtime": True, + "results": [], + "posted_to_telegram": False, + "telegram_transport_deny": { + "enabled": True, + "attempt_count": 0, + "patched_methods": sorted(protocol_lib.EXPECTED_TELEGRAM_DENY_METHODS), + "expected_methods": sorted(protocol_lib.EXPECTED_TELEGRAM_DENY_METHODS), + "runner_adapters_empty": True, + }, + "db_counts_before": counts, + "db_counts_after": counts, + "db_counts_changed": False, + "db_fingerprint_before": fingerprint, + "db_fingerprint_after": fingerprint, + "db_fingerprint_unchanged": True, + "preexecution_safety_gate": {"status": "pass"}, + "temp_profile_removed": True, + "post_run_orphan_readback": {"no_matching_processes": True}, + "before_service": service_before if before_restart else service_after, + "service_before_after": { + "after": service_before if before_restart else service_after, + }, + } + path.write_text(json.dumps(payload, sort_keys=True) + "\n", encoding="utf-8") + return {"path": str(path), "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), "pass": True} + + directory.mkdir(parents=True, exist_ok=True) + before_probe = probe(directory / "restart-before.json", before_restart=True) + after_probe = probe(directory / "restart-after.json", before_restart=False) + return { + "schema": "livingip.leoGatewayRestartReceipt.v1", + "generated_at_utc": "2026-07-15T00:01:30+00:00", + "protocol_id": protocol["protocol_id"], + "protocol_hash_sha256": protocol["protocol_hash_sha256"], + "next_trial_id": trial["trial_id"], + "next_trial_prompt_set_sha256": trial["prompt_set_sha256"], + "restart_started_at_utc": "2026-07-15T00:00:30+00:00", + "restart_ended_at_utc": "2026-07-15T00:01:00+00:00", + "restart_returncode": 0, + "service_before": service_before, + "service_after": service_after, + "deploy_before": deploy, + "deploy_after": deploy, + "posted_to_telegram": False, + "db_counts_before": counts, + "db_counts_after": counts, + "db_counts_changed": False, + "db_fingerprint_before": fingerprint, + "db_fingerprint_after": fingerprint, + "db_fingerprint_unchanged": True, + "before_probe": before_probe, + "after_probe": after_probe, + "checks": {"all_independent_checks": True}, + "pass": True, + } + + +def score_for_trial(protocol: dict, trial: dict, artifact_directory: Path | None = None) -> dict: + temporary = tempfile.TemporaryDirectory() if artifact_directory is None else None + raw_directory = Path(temporary.name) if temporary is not None else artifact_directory + assert raw_directory is not None + raw_directory.mkdir(parents=True, exist_ok=True) + grounded = fake_report(protocol, trial, grounded=True) + baseline = fake_report(protocol, trial, grounded=False) + grounded_path = raw_directory / "grounded.json" + baseline_path = raw_directory / "baseline.json" + grounded_path.write_text(json.dumps(grounded, sort_keys=True) + "\n", encoding="utf-8") + baseline_path.write_text(json.dumps(baseline, sort_keys=True) + "\n", encoding="utf-8") + restart = ( + restart_receipt(protocol, raw_directory / "restart", trial) + if trial["session_mode"] == "post_restart_clean_session" + else None + ) + restart_path = raw_directory / "restart-receipt.json" + if restart is not None: + restart_path.write_text(json.dumps(restart, sort_keys=True) + "\n", encoding="utf-8") + try: + score = protocol_lib.score_live_trial( + protocol, + trial["trial_id"], + grounded, + baseline_report=baseline, + restart_receipt=restart, + ) + score["grounded_report_path"] = str(grounded_path) + score["baseline_report_path"] = str(baseline_path) + score["grounded_report_sha256"] = hashlib.sha256(grounded_path.read_bytes()).hexdigest() + score["baseline_report_sha256"] = hashlib.sha256(baseline_path.read_bytes()).hexdigest() + if restart is not None: + score["restart_receipt_path"] = str(restart_path) + score["restart_receipt_sha256"] = hashlib.sha256(restart_path.read_bytes()).hexdigest() + score["restart_receipt_payload_sha256"] = protocol_lib.canonical_sha256(restart) + return score + finally: + if temporary is not None: + temporary.cleanup() + + +def test_protocol_freezes_three_unique_variants_and_follow_up_shapes() -> None: + protocol = frozen_protocol() + assert protocol_lib.validate_protocol(protocol, verify_source_hashes=True)["pass"] is True + assert len(protocol["trials"]) == 3 + assert protocol["trials"][-1]["session_mode"] == "post_restart_clean_session" + for family_id in protocol["blinding"]["prompt_families"]: + prompts = [ + next(item for item in trial["prompts"] if item["family_id"] == family_id) + for trial in protocol["trials"] + ] + assert len({item["variant_index"] for item in prompts}) == 3 + assert len({item["expected_follow_up"] for item in prompts}) == 3 + assert all(item["leakage_markers"] for item in prompts) + assert all("row id:" not in item["message"].lower() for item in prompts) + + +def test_protocol_validation_rejects_prompt_and_source_hash_tampering() -> None: + protocol = frozen_protocol() + protocol["trials"][0]["prompts"][0]["message"] += " changed" + protocol["protocol_hash_sha256"] = protocol_lib.canonical_sha256( + {key: value for key, value in protocol.items() if key != "protocol_hash_sha256"} + ) + assert "prompt_hash_mismatch:" in " ".join( + protocol_lib.validate_protocol(protocol, verify_source_hashes=True)["issues"] + ) + + protocol = frozen_protocol() + protocol["source_hashes"]["readonly_guard_sha256"] = "0" * 64 + protocol["protocol_hash_sha256"] = protocol_lib.canonical_sha256( + {key: value for key, value in protocol.items() if key != "protocol_hash_sha256"} + ) + assert "source_changed_after_freeze:readonly_guard_sha256" in protocol_lib.validate_protocol( + protocol, verify_source_hashes=True + )["issues"] + + +def test_live_trial_requires_semantics_subject_receipts_and_real_ablation() -> None: + protocol = frozen_protocol() + trial = protocol["trials"][0] + score = score_for_trial(protocol, trial) + assert score["pass"] is True + assert score["grounded_pass_rate"] == 1.0 + assert score["receipt_ablation"]["grounded_pass_rate"] == 0.0 + assert all(item["receipt_pass"] for item in score["prompt_scores"]) + assert all(score["comparison_axis_checks"].values()) + + +def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> None: + protocol = frozen_protocol() + trial = protocol["trials"][0] + for grounded in (True, False): + report = fake_report(protocol, trial, grounded=grounded) + assert protocol_lib._benchmark_execution_chain(report)["pass"] is True + + manifest = report["results"][0]["execution_manifest"] + manifest["attribution"]["missing_required_bindings"].append("unexpected_gap") + stable = { + key: value + for key, value in manifest.items() + if key not in {"generated_at_utc", "execution_sha256"} + } + manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable) + validation = protocol_lib._benchmark_execution_chain(report) + assert validation["pass"] is False + assert validation["turn_checks"][trial["prompts"][0]["id"]]["declared_missing_exact"] is False + + +def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal() -> None: + protocol = frozen_protocol() + trial = protocol["trials"][0] + grounded = fake_report(protocol, trial, grounded=True) + baseline = fake_report(protocol, trial, grounded=False) + behavior = baseline["executed_behavior_manifest"] + identity = behavior["components"]["runtime_identity"]["content"] + identity["files"][0]["sha256"] = "9" * 64 + identity_stable = { + "files": identity["files"], + "missing": identity["missing"], + "symlinks": identity["symlinks"], + } + identity["sha256"] = protocol_lib.canonical_sha256(identity_stable) + behavior_stable = { + key: behavior[key] + for key in ( + "schema", + "model_runtime", + "hermes_runtime", + "teleo_infrastructure_runtime", + "components", + "canonical_database", + ) + } + behavior["behavior_sha256"] = protocol_lib.canonical_sha256(behavior_stable) + for result in baseline["results"]: + manifest = result["execution_manifest"] + manifest["runtime"]["behavior_sha256"] = behavior["behavior_sha256"] + stable = { + key: value + for key, value in manifest.items() + if key not in {"generated_at_utc", "execution_sha256"} + } + manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable) + for previous, result in zip(baseline["results"], baseline["results"][1:], strict=False): + manifest = result["execution_manifest"] + manifest["session_boundary"]["conversation"]["previous_execution_sha256"] = previous[ + "execution_manifest" + ]["execution_sha256"] + stable = { + key: value + for key, value in manifest.items() + if key not in {"generated_at_utc", "execution_sha256"} + } + manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable) + + score = protocol_lib.score_live_trial( + protocol, + trial["trial_id"], + grounded, + baseline_report=baseline, + ) + assert score["pass"] is False + assert score["executed_behavior_ablation"]["checks"]["non_middleware_components_equal"] is False + assert ( + score["comparison_axis_checks"]["executed_behavior_manifests_capture_only_declared_ablation"] + is False + ) + + +def test_relative_retained_paths_resolve_from_repo_root() -> None: + assert protocol_lib._retained_path("pyproject.toml") == ROOT / "pyproject.toml" + + +def test_live_trial_rejects_duplicate_results_malformed_receipts_and_invented_ids() -> None: + protocol = frozen_protocol() + trial = protocol["trials"][0] + grounded = fake_report(protocol, trial, grounded=True) + grounded["results"].append(copy.deepcopy(grounded["results"][0])) + score = protocol_lib.score_live_trial( + protocol, + trial["trial_id"], + grounded, + baseline_report=fake_report(protocol, trial, grounded=False), + ) + assert score["pass"] is False + assert score["prompt_binding"]["checks"]["prompt_ids_unique"] is False + + grounded = fake_report(protocol, trial, grounded=True) + grounded["results"][0]["database_context_trace"][0]["retrieval_receipt"]["receipt_sha256"] = "broken" + grounded["results"][0]["reply"] += " Unsupported row 12345678-1234-4123-8123-123456789abc." + score = protocol_lib.score_live_trial( + protocol, + trial["trial_id"], + grounded, + baseline_report=fake_report(protocol, trial, grounded=False), + ) + first = score["prompt_scores"][0] + assert score["pass"] is False + assert first["receipt_pass"] is False + assert first["receipt_score"]["unsupported_identifiers"] == [ + "12345678-1234-4123-8123-123456789abc" + ] + + grounded = fake_report(protocol, trial, grounded=True) + traced_receipt = grounded["results"][0]["database_context_trace"][0]["retrieval_receipt"] + traced_receipt["counts"]["claims"] += 1 + score = protocol_lib.score_live_trial( + protocol, + trial["trial_id"], + grounded, + baseline_report=fake_report(protocol, trial, grounded=False), + ) + assert score["pass"] is False + assert score["prompt_scores"][0]["receipt_pass"] is False + + grounded = fake_report(protocol, trial, grounded=True) + replayed_hash = "f" * 64 + for item in grounded["results"][0]["database_context_trace"]: + item["query_sha256"] = replayed_hash + if item.get("retrieval_receipt"): + item["retrieval_receipt"]["query_sha256"] = replayed_hash + score = protocol_lib.score_live_trial( + protocol, + trial["trial_id"], + grounded, + baseline_report=fake_report(protocol, trial, grounded=False), + ) + first = score["prompt_scores"][0] + assert first["receipt_score"]["checks"]["context_response_query_hash_bound"] is False + assert score["pass"] is False + + grounded = fake_report(protocol, trial, grounded=True) + for item in grounded["results"][0]["database_context_trace"]: + item["contract_ids"] = "demo_capability_readback" + score = protocol_lib.score_live_trial( + protocol, + trial["trial_id"], + grounded, + baseline_report=fake_report(protocol, trial, grounded=False), + ) + assert score["prompt_scores"][0]["receipt_score"]["checks"]["contract_ids_are_typed_lists"] is False + + +def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words() -> None: + protocol = frozen_protocol() + source_prompt = next( + item for item in protocol["trials"][0]["prompts"] if item["family_id"] == "source_evidence" + ) + wrong_subject = "The database proposal is approved but not applied, so wait for a canonical receipt." + assert protocol_lib._subject_alignment(source_prompt, wrong_subject) is False + generic_family_only = "The source document has claim_evidence and evidence, but the named packet is omitted." + assert protocol_lib._subject_alignment(source_prompt, generic_family_only) is False + duplicate_subject = ( + f"{source_prompt['subject']} uses source evidence and claim_evidence; " + f"repeat {source_prompt['subject']} is not acceptable." + ) + assert protocol_lib._subject_alignment(source_prompt, duplicate_subject) is False + sibling_subject = next(item for item in source_prompt["family_subjects"] if item != source_prompt["subject"]) + shotgun = ( + f"{source_prompt['subject']} and {sibling_subject} both use source evidence and claim_evidence." + ) + assert protocol_lib._subject_alignment(source_prompt, shotgun) is False + + +def test_evidence_probe_four_line_reply_binds_exact_subject_and_receipt() -> None: + protocol = frozen_protocol() + trial = protocol["trials"][0] + prompt = next(item for item in trial["prompts"] if item["family_id"] == "receipt_discrimination") + result = result_for(prompt, trial["memory_token"], 1, grounded=True) + assert result["reply"] == ( + f"Subject: {prompt['subject']}\nMode: read-only\nSurface: context\nReceipt: aaaaaaaaaaaa" + ) + assert protocol_lib._subject_alignment(prompt, result["reply"]) is True + semantic = protocol_lib._score_semantic_results([result], {**trial, "prompts": [prompt]}) + assert semantic["pass"] is True + evidence = protocol_lib._evidence_answer_score( + prompt, + result, + semantic_pass=True, + subject_alignment=True, + grounded_tool_hashes=["a" * 64], + ) + assert evidence["pass"] is True + + sibling = next(item for item in prompt["family_subjects"] if item != prompt["subject"]) + result["reply"] += f"\nSubject: {sibling}" + assert protocol_lib._subject_alignment(prompt, result["reply"]) is False + + +def test_restart_receipt_binds_new_pid_full_fingerprint_and_next_trial(tmp_path: Path) -> None: + protocol = frozen_protocol() + trial = protocol["trials"][-1] + report = fake_report(protocol, trial, grounded=True) + valid = protocol_lib.validate_restart_receipt(restart_receipt(protocol, tmp_path, trial), report) + assert valid["pass"] is True + broken = restart_receipt(protocol, tmp_path / "broken-pid", trial) + broken["service_after"]["MainPID"] = "101" + assert protocol_lib.validate_restart_receipt(broken, report)["pass"] is False + + wrong_protocol = restart_receipt(protocol, tmp_path / "wrong-protocol", trial) + wrong_protocol["protocol_hash_sha256"] = "0" * 64 + validation = protocol_lib.validate_restart_receipt(wrong_protocol, report) + assert validation["pass"] is False + assert validation["checks"]["protocol_hash_bound"] is False + + missing_fingerprint = restart_receipt(protocol, tmp_path / "missing-fingerprint", trial) + missing_fingerprint["db_fingerprint_before"] = {"status": "ok"} + assert protocol_lib.validate_restart_receipt(missing_fingerprint, report)["pass"] is False + + corrupt_probe = restart_receipt(protocol, tmp_path / "corrupt-probe", trial) + corrupt_probe["before_probe"]["sha256"] = "0" * 64 + validation = protocol_lib.validate_restart_receipt(corrupt_probe, report) + assert validation["checks"]["before_probe_artifact_valid"] is False + assert validation["pass"] is False + + +def test_identical_grounded_and_ablation_replies_cannot_create_evidence_delta() -> None: + protocol = frozen_protocol() + trial = protocol["trials"][0] + grounded = fake_report(protocol, trial, grounded=True) + ablated = fake_report(protocol, trial, grounded=False) + grounded_by_prompt = {item["prompt_id"]: item for item in grounded["results"]} + for result in ablated["results"]: + result["reply"] = grounded_by_prompt[result["prompt_id"]]["reply"] + score = protocol_lib.score_live_trial(protocol, trial["trial_id"], grounded, baseline_report=ablated) + comparison = score["evidence_answer_comparison"] + assert comparison["current_pass_rate"] == comparison["ablation_pass_rate"] + assert comparison["current_minus_ablation_delta"] == 0.0 + assert score["pass"] is False + + +def test_receipt_discriminator_rejects_wrong_command_extra_call_and_wrong_token() -> None: + protocol = frozen_protocol() + trial = protocol["trials"][0] + evidence_prompt = next(item for item in trial["prompts"] if item["custom_evidence_probe"]) + + for mutation in ("wrong_command", "extra_call", "wrong_token"): + grounded = fake_report(protocol, trial, grounded=True) + result = next(item for item in grounded["results"] if item["prompt_id"] == evidence_prompt["id"]) + trace = result["database_tool_trace"] + if mutation == "wrong_command": + trace["calls"][0]["database_invocations"][0]["command_sha256"] = "f" * 64 + elif mutation == "extra_call": + trace["calls"].append(copy.deepcopy(trace["calls"][0])) + trace["database_tool_call_count"] = 2 + trace["database_tool_completed_count"] = 2 + else: + result["reply"] = result["reply"].replace("Receipt: aaaaaaaaaaaa", "Receipt: bbbbbbbbbbbb") + post = result["database_context_trace"][-1] + post["delivered_response_sha256"] = hashlib.sha256(result["reply"].encode()).hexdigest() + score = protocol_lib.score_live_trial( + protocol, + trial["trial_id"], + grounded, + baseline_report=fake_report(protocol, trial, grounded=False), + ) + prompt_score = next(item for item in score["prompt_scores"] if item["prompt_id"] == evidence_prompt["id"]) + assert prompt_score["evidence_answer_pass"] is False, mutation + assert score["pass"] is False, mutation + + +def test_aggregate_recomputes_integrity_and_rejects_forged_minimal_scores(tmp_path: Path) -> None: + protocol = frozen_protocol() + scores = [ + score_for_trial(protocol, trial, tmp_path / trial["trial_id"]) + for trial in protocol["trials"] + ] + aggregate = protocol_lib.aggregate_trial_scores(protocol, scores) + assert aggregate["pass"] is True + assert aggregate["mean_current_grounded_pass_rate"] == 1.0 + assert aggregate["mean_ablation_grounded_pass_rate"] == 0.0 + + forged = [ + { + "trial_id": trial["trial_id"], + "pass": True, + "grounded_pass_rate": 1.0, + "receipt_ablation": {"grounded_pass_rate": 0.0}, + "restart_receipt_validation": {"pass": True}, + } + for trial in protocol["trials"] + ] + rejected = protocol_lib.aggregate_trial_scores(protocol, forged) + assert rejected["pass"] is False + assert rejected["checks"]["all_trial_scores_integrity_valid"] is False + + tampered = copy.deepcopy(scores) + tampered[0]["comparison_axis_checks"]["same_model_identity"] = False + rejected = protocol_lib.aggregate_trial_scores(protocol, tampered) + assert rejected["pass"] is False + assert ( + rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]]["checks"] + ["comparison_axis_checks_passed"] + is False + ) + + forged_from_failing_report = copy.deepcopy(scores) + report_path = Path(forged_from_failing_report[0]["grounded_report_path"]) + failing_report = json.loads(report_path.read_text(encoding="utf-8")) + failing_report["results"][0]["reply"] = "This answer is unrelated and ungrounded." + report_path.write_text(json.dumps(failing_report, sort_keys=True) + "\n", encoding="utf-8") + forged_from_failing_report[0]["grounded_report_sha256"] = hashlib.sha256(report_path.read_bytes()).hexdigest() + forged_from_failing_report[0]["grounded_report_payload_sha256"] = protocol_lib.canonical_sha256( + failing_report + ) + forged_from_failing_report[0]["derivation_core_sha256"] = protocol_lib.canonical_sha256( + protocol_lib.score_derivation_core(forged_from_failing_report[0]) + ) + rejected = protocol_lib.aggregate_trial_scores(protocol, forged_from_failing_report) + first_integrity = rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]] + assert rejected["pass"] is False + assert first_integrity["checks"]["grounded_report_artifact_bound"] is True + assert first_integrity["checks"]["score_recomputed_from_retained_artifacts"] is False + + report_path.write_text("{}\n", encoding="utf-8") + rejected = protocol_lib.aggregate_trial_scores(protocol, forged_from_failing_report) + assert rejected["pass"] is False + assert ( + rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]]["checks"] + ["grounded_report_artifact_bound"] + is False + )