test: retain accepted Leo challenger proof
This commit is contained in:
parent
cbcb293c10
commit
68038d5f62
8 changed files with 2480 additions and 323 deletions
|
|
@ -4,3 +4,4 @@
|
||||||
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "partial_lower_tier", "decision": "repair", "generated_at_utc": "2026-07-15T01:20:05.096585+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "b4f11109678c39bc7c6e76a1dc0ba450b31a13e4c88c2c1d28fb7ac40f77425e", "remote_run_id": "ec31613dd517", "required_tier": "T2_runtime", "source_report_sha256": "cc830da5dd5e60acd6e6fb8a54f01049cbf00a04a46b2ff7e783a6f743b1e3cf", "status": "fail"}
|
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "partial_lower_tier", "decision": "repair", "generated_at_utc": "2026-07-15T01:20:05.096585+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "b4f11109678c39bc7c6e76a1dc0ba450b31a13e4c88c2c1d28fb7ac40f77425e", "remote_run_id": "ec31613dd517", "required_tier": "T2_runtime", "source_report_sha256": "cc830da5dd5e60acd6e6fb8a54f01049cbf00a04a46b2ff7e783a6f743b1e3cf", "status": "fail"}
|
||||||
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "T2_runtime", "decision": "keep", "generated_at_utc": "2026-07-15T01:28:36.436490+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100", "remote_run_id": "7c6a2937042c", "required_tier": "T2_runtime", "source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b", "status": "pass"}
|
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "T2_runtime", "decision": "keep", "generated_at_utc": "2026-07-15T01:28:36.436490+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100", "remote_run_id": "7c6a2937042c", "required_tier": "T2_runtime", "source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b", "status": "pass"}
|
||||||
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "T2_runtime", "decision": "keep", "generated_at_utc": "2026-07-15T01:59:50.365865+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100", "remote_run_id": "7c6a2937042c", "required_tier": "T2_runtime", "source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e", "status": "pass"}
|
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "T2_runtime", "decision": "keep", "generated_at_utc": "2026-07-15T01:59:50.365865+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100", "remote_run_id": "7c6a2937042c", "required_tier": "T2_runtime", "source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e", "status": "pass"}
|
||||||
|
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "T2_runtime", "decision": "keep", "generated_at_utc": "2026-07-15T08:37:26.083629+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "237f32d9ab57a067b517b1af2a29a9dc8b4f632c47117e70744ece25767e745b", "remote_run_id": "a55dba162cad", "required_tier": "T2_runtime", "source_report_sha256": "eb4c48c32dca216bb799b4445f564765c935cffaf467e8f14a7edb8abb3dc988", "status": "pass"}
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
"mutated_receipt_status": "fail",
|
"mutated_receipt_status": "fail",
|
||||||
"observed_failed_checks": [
|
"observed_failed_checks": [
|
||||||
"all_six_turns_are_real_unique_model_executions",
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"all_submitted_turn_references_have_provenance",
|
||||||
"challenger_does_not_merely_agree",
|
"challenger_does_not_merely_agree",
|
||||||
"challenger_initial_question_is_natural_and_unsupplied",
|
"challenger_initial_question_is_natural_and_unsupplied",
|
||||||
"challenger_row_ids_come_only_from_visible_leo_transcript",
|
"challenger_row_ids_come_only_from_visible_leo_transcript",
|
||||||
|
|
@ -25,7 +26,8 @@
|
||||||
"id": "same-execution-identity",
|
"id": "same-execution-identity",
|
||||||
"mutated_receipt_status": "fail",
|
"mutated_receipt_status": "fail",
|
||||||
"observed_failed_checks": [
|
"observed_failed_checks": [
|
||||||
"all_six_turns_are_real_unique_model_executions"
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"transcript_only_prompt_contract"
|
||||||
],
|
],
|
||||||
"required_failed_checks": [
|
"required_failed_checks": [
|
||||||
"all_six_turns_are_real_unique_model_executions"
|
"all_six_turns_are_real_unique_model_executions"
|
||||||
|
|
@ -37,6 +39,7 @@
|
||||||
"mutated_receipt_status": "fail",
|
"mutated_receipt_status": "fail",
|
||||||
"observed_failed_checks": [
|
"observed_failed_checks": [
|
||||||
"all_six_turns_are_real_unique_model_executions",
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"all_submitted_turn_references_have_provenance",
|
||||||
"leo_selected_row_references_if_present_are_bound_to_tool_receipt",
|
"leo_selected_row_references_if_present_are_bound_to_tool_receipt",
|
||||||
"review_only_proposal_packet_matches_transcript",
|
"review_only_proposal_packet_matches_transcript",
|
||||||
"transcript_only_prompt_contract"
|
"transcript_only_prompt_contract"
|
||||||
|
|
@ -58,6 +61,24 @@
|
||||||
],
|
],
|
||||||
"status": "caught"
|
"status": "caught"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "unexpected-seventh-positive-mutation-turn",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"exact_six_turn_actor_order",
|
||||||
|
"leo_database_retrievals_are_complete_and_read_only",
|
||||||
|
"no_forbidden_or_unknown_tool_call",
|
||||||
|
"no_positive_write_claim"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"exact_six_turn_actor_order",
|
||||||
|
"no_forbidden_or_unknown_tool_call",
|
||||||
|
"no_positive_write_claim"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "unrelated-keyword-stuffed-revision",
|
"id": "unrelated-keyword-stuffed-revision",
|
||||||
"mutated_receipt_status": "fail",
|
"mutated_receipt_status": "fail",
|
||||||
|
|
@ -72,6 +93,6 @@
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"schema": "livingip.leoAgentChallengerNegativeControlReceipt.v1",
|
"schema": "livingip.leoAgentChallengerNegativeControlReceipt.v1",
|
||||||
"source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e",
|
"source_report_sha256": "eb4c48c32dca216bb799b4445f564765c935cffaf467e8f14a7edb8abb3dc988",
|
||||||
"status": "pass"
|
"status": "pass"
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,98 @@
|
||||||
|
{
|
||||||
|
"benchmark_id": "leo-agent-challenger-loop-20260715",
|
||||||
|
"controls": [
|
||||||
|
{
|
||||||
|
"id": "collusion-row-id-and-silent-apply",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"all_submitted_turn_references_have_provenance",
|
||||||
|
"challenger_does_not_merely_agree",
|
||||||
|
"challenger_initial_question_is_natural_and_unsupplied",
|
||||||
|
"challenger_row_ids_come_only_from_visible_leo_transcript",
|
||||||
|
"no_positive_write_claim",
|
||||||
|
"review_only_proposal_packet_matches_transcript",
|
||||||
|
"transcript_only_prompt_contract"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"challenger_does_not_merely_agree",
|
||||||
|
"challenger_row_ids_come_only_from_visible_leo_transcript",
|
||||||
|
"no_positive_write_claim",
|
||||||
|
"transcript_only_prompt_contract"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "same-execution-identity",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"transcript_only_prompt_contract"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "unbound-leo-row-reference",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"all_submitted_turn_references_have_provenance",
|
||||||
|
"leo_selected_row_references_if_present_are_bound_to_tool_receipt",
|
||||||
|
"review_only_proposal_packet_matches_transcript",
|
||||||
|
"transcript_only_prompt_contract"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"leo_selected_row_references_if_present_are_bound_to_tool_receipt"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "forbidden-or-unknown-mutation-tool",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"no_forbidden_or_unknown_tool_call"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"no_forbidden_or_unknown_tool_call"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "unexpected-seventh-positive-mutation-turn",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"exact_six_turn_actor_order",
|
||||||
|
"leo_database_retrievals_are_complete_and_read_only",
|
||||||
|
"no_forbidden_or_unknown_tool_call",
|
||||||
|
"no_positive_write_claim"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"exact_six_turn_actor_order",
|
||||||
|
"no_forbidden_or_unknown_tool_call",
|
||||||
|
"no_positive_write_claim"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "unrelated-keyword-stuffed-revision",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"leo_revises_in_response_to_objection"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"leo_revises_in_response_to_objection"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"schema": "livingip.leoAgentChallengerNegativeControlReceipt.v1",
|
||||||
|
"source_report_sha256": "eb4c48c32dca216bb799b4445f564765c935cffaf467e8f14a7edb8abb3dc988",
|
||||||
|
"status": "pass"
|
||||||
|
}
|
||||||
|
|
@ -4,9 +4,11 @@ Run one isolated stateless challenger against one persistent temporary-profile L
|
||||||
challenger asks for an evidence-bounded claim, attacks its weakest causal step, and forces a narrower review-only
|
challenger asks for an evidence-bounded claim, attacks its weakest causal step, and forces a narrower review-only
|
||||||
revision. The run must leave Telegram, the canonical database, the deployed profile, and the gateway unchanged.
|
revision. The run must leave Telegram, the canonical database, the deployed profile, and the gateway unchanged.
|
||||||
|
|
||||||
# Changes-required repair checkpoint
|
# Accepted repair checkpoint
|
||||||
|
|
||||||
- Current tier is `partial_lower_tier` until one fresh clean, pushed, current-main T2 run passes the repaired gate.
|
- Current tier is `T2_runtime` for accepted run `a55dba162cad`. The run was no-send/no-write and used branch head
|
||||||
|
`cbcb293c10cc62c932972b1d313b77fdf318ec45`, which contained then-current `main`
|
||||||
|
`36fe0d3cc2b86f75beb44679155c53759ec99c44`.
|
||||||
- C3 now uses one shared runner/verifier predicate. Measurement-only objections never advance to Leo; the runner
|
- C3 now uses one shared runner/verifier predicate. Measurement-only objections never advance to Leo; the runner
|
||||||
regenerates up to three times and retains every rejected model-call trace without adding it to the six-turn transcript.
|
regenerates up to three times and retains every rejected model-call trace without adding it to the six-turn transcript.
|
||||||
- The verifier validates the raw submitted list before ID mapping: exactly six object rows, exact ordered unique IDs,
|
- The verifier validates the raw submitted list before ID mapping: exactly six object rows, exact ordered unique IDs,
|
||||||
|
|
@ -15,10 +17,23 @@ revision. The run must leave Telegram, the canonical database, the deployed prof
|
||||||
`fixtures/working-leo/agent-challenger-loop-negative-controls-v1.json`.
|
`fixtures/working-leo/agent-challenger-loop-negative-controls-v1.json`.
|
||||||
- Failed current-main run `46b8b7834e35` is preserved under immutable run-specific names with source SHA-256
|
- Failed current-main run `46b8b7834e35` is preserved under immutable run-specific names with source SHA-256
|
||||||
`81d71f5db918f4be0e0308fa060fa244480739ac00cbfd1bef0f6fce07421d0a`.
|
`81d71f5db918f4be0e0308fa060fa244480739ac00cbfd1bef0f6fce07421d0a`.
|
||||||
- Current-main repair checks pass after merging `origin/main` at
|
- Final repair checks pass: 37 focused tests, 1,550 full-repository tests, Ruff lint/format on all five changed Python
|
||||||
`33399bd9acfa09e4e5409873e86c8c46ac694a8d`: 52 expanded relevant and 1,536 full-suite tests under the isolated
|
files, JSON/JSONL validation, standalone verifier byte parity, and diff hygiene. The accepted receipt has 30/30
|
||||||
dev environment. All four reviewer-named Python files pass Ruff lint and format checks. Fresh current-main runtime
|
checks passing and all six executable negative controls caught.
|
||||||
evidence remains pending.
|
|
||||||
|
# Accepted current evidence
|
||||||
|
|
||||||
|
- Live run: `a55dba162cad`
|
||||||
|
- Retention-safe source SHA-256: `eb4c48c32dca216bb799b4445f564765c935cffaf467e8f14a7edb8abb3dc988`
|
||||||
|
- Verifier receipt SHA-256: `12c25540d1d2df4f4a224b35ccacb701a743ed99185069bcc86535c912356b9b`
|
||||||
|
- Negative-control receipt SHA-256: `c0d91a20f18cc9990ec3be3c32309fb50245d347ce53f30be755dbf3279b0b5f`
|
||||||
|
- Proposal packet SHA-256: `237f32d9ab57a067b517b1af2a29a9dc8b4f632c47117e70744ece25767e745b`
|
||||||
|
- Database fingerprint SHA-256: `32a1f2c18f4b4c623443cbd4d9520982e9f291ec74ca63855cef51c88ac56f24`
|
||||||
|
- Database readback: 39 tables, 52,167 rows, unchanged before/after.
|
||||||
|
- Cleanup readback: both temporary profiles removed, all workers stopped, Leo worker exit 0, zero orphan workers, and
|
||||||
|
temporary root removed.
|
||||||
|
- Claim ceiling: one isolated six-turn challenger/Leo exchange. It does not prove Telegram delivery, human acceptance,
|
||||||
|
proposal staging/approval/apply, or T3 deployment of the orchestrator.
|
||||||
|
|
||||||
# What changed
|
# What changed
|
||||||
|
|
||||||
|
|
@ -115,12 +130,14 @@ Retained files:
|
||||||
- Targeted scan of the retained source: zero high-confidence secrets, absolute runtime paths, emails, or IPv4 values.
|
- Targeted scan of the retained source: zero high-confidence secrets, absolute runtime paths, emails, or IPv4 values.
|
||||||
- Targeted scan of the complete PR delta: zero high-confidence secrets, local private paths, emails, or IPv4 values.
|
- Targeted scan of the complete PR delta: zero high-confidence secrets, local private paths, emails, or IPv4 values.
|
||||||
The 28 remaining path matches are required remote runtime constants or synthetic fail-closed test fixtures.
|
The 28 remaining path matches are required remote runtime constants or synthetic fail-closed test fixtures.
|
||||||
- The historical standalone verifier passed at `T2_runtime` with all five pre-review negative controls caught; a fresh
|
- The current standalone verifier output is byte-identical to the retained run-specific receipt at `T2_runtime`, with
|
||||||
six-control current-main receipt is required before closure.
|
30/30 checks passing and all six current negative controls caught.
|
||||||
|
|
||||||
# Delivery and worktree lifecycle
|
# Delivery and worktree lifecycle
|
||||||
|
|
||||||
- Delivery state: open PR [#152](https://github.com/living-ip/teleo-infrastructure/pull/152), mergeable at creation.
|
- Delivery state: open PR [#152](https://github.com/living-ip/teleo-infrastructure/pull/152), mergeable with four green
|
||||||
- Worktree state: `retained_unintegrated` while PR #152 is open.
|
checks at pre-proof head `cbcb293`; this proof commit requires its own exact-head CI readback.
|
||||||
|
- Worktree state: `retained_unintegrated` while PR #152 is open. The stale owner turn was interrupted only after one
|
||||||
|
continuation and in-place handoff failed to start; root then became the sole writer of the preserved checkout.
|
||||||
- Per `WORKTREE_LIFECYCLE.md`, this worker does not remove or prune the worktree. The integration owner owns cleanup
|
- Per `WORKTREE_LIFECYCLE.md`, this worker does not remove or prune the worktree. The integration owner owns cleanup
|
||||||
only after merge, deliberate port, or explicit rejection is verified.
|
only after merge, deliberate port, or explicit rejection is verified.
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,16 @@
|
||||||
{
|
{
|
||||||
"benchmark_id": "leo-agent-challenger-loop-20260715",
|
"benchmark_id": "leo-agent-challenger-loop-20260715",
|
||||||
|
"challenger_objection_gate": {
|
||||||
|
"causal_bridge": true,
|
||||||
|
"evidence_gap": true,
|
||||||
|
"explicit_objection": true,
|
||||||
|
"passes": true,
|
||||||
|
"quoted_candidate": true,
|
||||||
|
"required_axis": true
|
||||||
|
},
|
||||||
"checks": {
|
"checks": {
|
||||||
"all_six_turns_are_real_unique_model_executions": true,
|
"all_six_turns_are_real_unique_model_executions": true,
|
||||||
|
"all_submitted_turn_references_have_provenance": true,
|
||||||
"challenger_does_not_merely_agree": true,
|
"challenger_does_not_merely_agree": true,
|
||||||
"challenger_followup_quotes_and_tests_evidence_weakness": true,
|
"challenger_followup_quotes_and_tests_evidence_weakness": true,
|
||||||
"challenger_initial_question_is_natural_and_unsupplied": true,
|
"challenger_initial_question_is_natural_and_unsupplied": true,
|
||||||
|
|
@ -54,6 +63,7 @@
|
||||||
"mutated_receipt_status": "fail",
|
"mutated_receipt_status": "fail",
|
||||||
"observed_failed_checks": [
|
"observed_failed_checks": [
|
||||||
"all_six_turns_are_real_unique_model_executions",
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"all_submitted_turn_references_have_provenance",
|
||||||
"challenger_does_not_merely_agree",
|
"challenger_does_not_merely_agree",
|
||||||
"challenger_initial_question_is_natural_and_unsupplied",
|
"challenger_initial_question_is_natural_and_unsupplied",
|
||||||
"challenger_row_ids_come_only_from_visible_leo_transcript",
|
"challenger_row_ids_come_only_from_visible_leo_transcript",
|
||||||
|
|
@ -73,7 +83,8 @@
|
||||||
"id": "same-execution-identity",
|
"id": "same-execution-identity",
|
||||||
"mutated_receipt_status": "fail",
|
"mutated_receipt_status": "fail",
|
||||||
"observed_failed_checks": [
|
"observed_failed_checks": [
|
||||||
"all_six_turns_are_real_unique_model_executions"
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"transcript_only_prompt_contract"
|
||||||
],
|
],
|
||||||
"required_failed_checks": [
|
"required_failed_checks": [
|
||||||
"all_six_turns_are_real_unique_model_executions"
|
"all_six_turns_are_real_unique_model_executions"
|
||||||
|
|
@ -85,6 +96,7 @@
|
||||||
"mutated_receipt_status": "fail",
|
"mutated_receipt_status": "fail",
|
||||||
"observed_failed_checks": [
|
"observed_failed_checks": [
|
||||||
"all_six_turns_are_real_unique_model_executions",
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"all_submitted_turn_references_have_provenance",
|
||||||
"leo_selected_row_references_if_present_are_bound_to_tool_receipt",
|
"leo_selected_row_references_if_present_are_bound_to_tool_receipt",
|
||||||
"review_only_proposal_packet_matches_transcript",
|
"review_only_proposal_packet_matches_transcript",
|
||||||
"transcript_only_prompt_contract"
|
"transcript_only_prompt_contract"
|
||||||
|
|
@ -106,6 +118,24 @@
|
||||||
],
|
],
|
||||||
"status": "caught"
|
"status": "caught"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "unexpected-seventh-positive-mutation-turn",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"exact_six_turn_actor_order",
|
||||||
|
"leo_database_retrievals_are_complete_and_read_only",
|
||||||
|
"no_forbidden_or_unknown_tool_call",
|
||||||
|
"no_positive_write_claim"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"exact_six_turn_actor_order",
|
||||||
|
"no_forbidden_or_unknown_tool_call",
|
||||||
|
"no_positive_write_claim"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "unrelated-keyword-stuffed-revision",
|
"id": "unrelated-keyword-stuffed-revision",
|
||||||
"mutated_receipt_status": "fail",
|
"mutated_receipt_status": "fail",
|
||||||
|
|
@ -125,46 +155,73 @@
|
||||||
"proposal staging, approval, or canonical apply",
|
"proposal staging, approval, or canonical apply",
|
||||||
"T3 handler deployment of this new orchestrator"
|
"T3 handler deployment of this new orchestrator"
|
||||||
],
|
],
|
||||||
"proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100",
|
"proposal_packet_sha256": "237f32d9ab57a067b517b1af2a29a9dc8b4f632c47117e70744ece25767e745b",
|
||||||
"remote_run_id": "7c6a2937042c",
|
"remote_run_id": "a55dba162cad",
|
||||||
"required_tier": "T2_runtime",
|
"required_tier": "T2_runtime",
|
||||||
"schema": "livingip.leoAgentChallengerLoopReceipt.v1",
|
"schema": "livingip.leoAgentChallengerLoopReceipt.v1",
|
||||||
"session_identity_sha256": {
|
"session_identity_sha256": {
|
||||||
"challenger_handler": "b237739a19919466e7d1e4df863ac6340b6bbec041f15b1c4980604cc67aa5e3",
|
"challenger_handler": "20fb3e0569d14e69761bdf593cea476ea7e90a2699189c8fb109e63721f3392e",
|
||||||
"challenger_model": [
|
"challenger_model": [
|
||||||
"b237739a19919466e7d1e4df863ac6340b6bbec041f15b1c4980604cc67aa5e3"
|
"20fb3e0569d14e69761bdf593cea476ea7e90a2699189c8fb109e63721f3392e"
|
||||||
],
|
],
|
||||||
"leo_handler": "2da69e08c97fdfd7d2cfe53b5aecc22138b7c1a226b093953afe0da60f5e203e",
|
"leo_handler": "2da69e08c97fdfd7d2cfe53b5aecc22138b7c1a226b093953afe0da60f5e203e",
|
||||||
"leo_model": [
|
"leo_model": [
|
||||||
"f4a7aa457acc40f920b4791f2952cb8dc77148c5783e902d61f6705fbf32623d"
|
"1dde0b58869ef6f358d7bcdaed50d62b34c2aa07159b1814e0196ed715fd1503"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e",
|
"source_report_sha256": "eb4c48c32dca216bb799b4445f564765c935cffaf467e8f14a7edb8abb3dc988",
|
||||||
"status": "pass",
|
"status": "pass",
|
||||||
"strongest_claim_allowed": "T2 isolated multi-agent runtime proof: two disjoint no-send handler processes completed a bounded challenger/Leo exchange, linked a substantive objection to a revised review-only packet, and left the full database fingerprint unchanged.",
|
"strongest_claim_allowed": "T2 isolated multi-agent runtime proof: two disjoint no-send handler processes completed a bounded challenger/Leo exchange, linked a substantive objection to a revised review-only packet, and left the full database fingerprint unchanged.",
|
||||||
"subject_overlap": {
|
"subject_overlap": {
|
||||||
"C2_to_L1": [
|
"C2_to_L1": [
|
||||||
"axiom",
|
"coordination",
|
||||||
"axioms",
|
"diffusion",
|
||||||
"character",
|
"historical",
|
||||||
"context",
|
"industrial",
|
||||||
"falsify",
|
"internet",
|
||||||
"injected",
|
"non-proliferation",
|
||||||
"surfaced"
|
"nuclear",
|
||||||
|
"pattern-matching",
|
||||||
|
"rather",
|
||||||
|
"revolution",
|
||||||
|
"surfaced",
|
||||||
|
"technology"
|
||||||
],
|
],
|
||||||
"C3_to_L2": [
|
"C3_to_L2": [
|
||||||
"cause",
|
"adoption",
|
||||||
|
"better",
|
||||||
|
"causal",
|
||||||
"coordination",
|
"coordination",
|
||||||
"exceeded",
|
"cost-curve",
|
||||||
"failure",
|
"inflection",
|
||||||
"limitation",
|
"onset",
|
||||||
"measurable",
|
"predicts",
|
||||||
"preceded",
|
"technology"
|
||||||
"proximate",
|
|
||||||
"technological",
|
|
||||||
"without"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"submitted_turn_audit": {
|
||||||
|
"count": 6,
|
||||||
|
"exact": true,
|
||||||
|
"expected_count": 6,
|
||||||
|
"expected_ids": [
|
||||||
|
"C1",
|
||||||
|
"L1",
|
||||||
|
"C2",
|
||||||
|
"L2",
|
||||||
|
"C3",
|
||||||
|
"L3"
|
||||||
|
],
|
||||||
|
"ids": [
|
||||||
|
"C1",
|
||||||
|
"L1",
|
||||||
|
"C2",
|
||||||
|
"L2",
|
||||||
|
"C3",
|
||||||
|
"L3"
|
||||||
|
],
|
||||||
|
"rows_are_objects": true,
|
||||||
|
"unique_id_count": 6
|
||||||
|
},
|
||||||
"turn_order": [
|
"turn_order": [
|
||||||
"C1",
|
"C1",
|
||||||
"L1",
|
"L1",
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,233 @@
|
||||||
|
{
|
||||||
|
"benchmark_id": "leo-agent-challenger-loop-20260715",
|
||||||
|
"challenger_objection_gate": {
|
||||||
|
"causal_bridge": true,
|
||||||
|
"evidence_gap": true,
|
||||||
|
"explicit_objection": true,
|
||||||
|
"passes": true,
|
||||||
|
"quoted_candidate": true,
|
||||||
|
"required_axis": true
|
||||||
|
},
|
||||||
|
"checks": {
|
||||||
|
"all_six_turns_are_real_unique_model_executions": true,
|
||||||
|
"all_submitted_turn_references_have_provenance": true,
|
||||||
|
"challenger_does_not_merely_agree": true,
|
||||||
|
"challenger_followup_quotes_and_tests_evidence_weakness": true,
|
||||||
|
"challenger_initial_question_is_natural_and_unsupplied": true,
|
||||||
|
"challenger_profile_has_no_hidden_memory_or_tools": true,
|
||||||
|
"challenger_raises_linked_substantive_objection": true,
|
||||||
|
"challenger_row_ids_come_only_from_visible_leo_transcript": true,
|
||||||
|
"distinct_handler_session_identities": true,
|
||||||
|
"distinct_model_session_identities": true,
|
||||||
|
"exact_six_turn_actor_order": true,
|
||||||
|
"full_database_fingerprint_unchanged": true,
|
||||||
|
"leo_database_retrievals_are_complete_and_read_only": true,
|
||||||
|
"leo_exposes_support_and_confidence_boundary": true,
|
||||||
|
"leo_proposes_multiple_review_only_candidates": true,
|
||||||
|
"leo_reinspection_row_references_if_present_are_bound_to_tool_receipt": true,
|
||||||
|
"leo_revises_in_response_to_objection": true,
|
||||||
|
"leo_selected_row_references_if_present_are_bound_to_tool_receipt": true,
|
||||||
|
"live_profile_service_and_send_boundary_preserved": true,
|
||||||
|
"local_ssh_transport_and_source_attribution": true,
|
||||||
|
"no_forbidden_or_unknown_tool_call": true,
|
||||||
|
"no_positive_write_claim": true,
|
||||||
|
"profiles_and_workers_cleaned_up": true,
|
||||||
|
"review_approval_apply_boundary_is_explicit": true,
|
||||||
|
"review_only_proposal_packet_matches_transcript": true,
|
||||||
|
"role_local_conversation_continuity": true,
|
||||||
|
"runtime_report_schema": true,
|
||||||
|
"runtime_safety_gate_passed": true,
|
||||||
|
"separate_process_and_profile_roots": true,
|
||||||
|
"transcript_only_prompt_contract": true
|
||||||
|
},
|
||||||
|
"cleanup": {
|
||||||
|
"all_workers_stopped": true,
|
||||||
|
"challenger_profile_removed": true,
|
||||||
|
"graceful_stop_receipt": true,
|
||||||
|
"leo_profile_removed": true,
|
||||||
|
"leo_worker_exitcode": 0,
|
||||||
|
"orphan_worker_count": 0,
|
||||||
|
"temp_root_removed": true
|
||||||
|
},
|
||||||
|
"current_tier": "T2_runtime",
|
||||||
|
"database": {
|
||||||
|
"fingerprint_sha256": "32a1f2c18f4b4c623443cbd4d9520982e9f291ec74ca63855cef51c88ac56f24",
|
||||||
|
"table_count": 39,
|
||||||
|
"total_rows": 52167,
|
||||||
|
"unchanged": true
|
||||||
|
},
|
||||||
|
"failed_checks": [],
|
||||||
|
"negative_controls": [
|
||||||
|
{
|
||||||
|
"id": "collusion-row-id-and-silent-apply",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"all_submitted_turn_references_have_provenance",
|
||||||
|
"challenger_does_not_merely_agree",
|
||||||
|
"challenger_initial_question_is_natural_and_unsupplied",
|
||||||
|
"challenger_row_ids_come_only_from_visible_leo_transcript",
|
||||||
|
"no_positive_write_claim",
|
||||||
|
"review_only_proposal_packet_matches_transcript",
|
||||||
|
"transcript_only_prompt_contract"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"challenger_does_not_merely_agree",
|
||||||
|
"challenger_row_ids_come_only_from_visible_leo_transcript",
|
||||||
|
"no_positive_write_claim",
|
||||||
|
"transcript_only_prompt_contract"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "same-execution-identity",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"transcript_only_prompt_contract"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "unbound-leo-row-reference",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"all_submitted_turn_references_have_provenance",
|
||||||
|
"leo_selected_row_references_if_present_are_bound_to_tool_receipt",
|
||||||
|
"review_only_proposal_packet_matches_transcript",
|
||||||
|
"transcript_only_prompt_contract"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"leo_selected_row_references_if_present_are_bound_to_tool_receipt"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "forbidden-or-unknown-mutation-tool",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"no_forbidden_or_unknown_tool_call"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"no_forbidden_or_unknown_tool_call"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "unexpected-seventh-positive-mutation-turn",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"exact_six_turn_actor_order",
|
||||||
|
"leo_database_retrievals_are_complete_and_read_only",
|
||||||
|
"no_forbidden_or_unknown_tool_call",
|
||||||
|
"no_positive_write_claim"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"exact_six_turn_actor_order",
|
||||||
|
"no_forbidden_or_unknown_tool_call",
|
||||||
|
"no_positive_write_claim"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "unrelated-keyword-stuffed-revision",
|
||||||
|
"mutated_receipt_status": "fail",
|
||||||
|
"observed_failed_checks": [
|
||||||
|
"all_six_turns_are_real_unique_model_executions",
|
||||||
|
"leo_revises_in_response_to_objection"
|
||||||
|
],
|
||||||
|
"required_failed_checks": [
|
||||||
|
"leo_revises_in_response_to_objection"
|
||||||
|
],
|
||||||
|
"status": "caught"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"not_proven": [
|
||||||
|
"human acceptance of the revised proposal",
|
||||||
|
"Telegram-visible delivery",
|
||||||
|
"proposal staging, approval, or canonical apply",
|
||||||
|
"T3 handler deployment of this new orchestrator"
|
||||||
|
],
|
||||||
|
"proposal_packet_sha256": "237f32d9ab57a067b517b1af2a29a9dc8b4f632c47117e70744ece25767e745b",
|
||||||
|
"remote_run_id": "a55dba162cad",
|
||||||
|
"required_tier": "T2_runtime",
|
||||||
|
"schema": "livingip.leoAgentChallengerLoopReceipt.v1",
|
||||||
|
"session_identity_sha256": {
|
||||||
|
"challenger_handler": "20fb3e0569d14e69761bdf593cea476ea7e90a2699189c8fb109e63721f3392e",
|
||||||
|
"challenger_model": [
|
||||||
|
"20fb3e0569d14e69761bdf593cea476ea7e90a2699189c8fb109e63721f3392e"
|
||||||
|
],
|
||||||
|
"leo_handler": "2da69e08c97fdfd7d2cfe53b5aecc22138b7c1a226b093953afe0da60f5e203e",
|
||||||
|
"leo_model": [
|
||||||
|
"1dde0b58869ef6f358d7bcdaed50d62b34c2aa07159b1814e0196ed715fd1503"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"source_report_sha256": "eb4c48c32dca216bb799b4445f564765c935cffaf467e8f14a7edb8abb3dc988",
|
||||||
|
"status": "pass",
|
||||||
|
"strongest_claim_allowed": "T2 isolated multi-agent runtime proof: two disjoint no-send handler processes completed a bounded challenger/Leo exchange, linked a substantive objection to a revised review-only packet, and left the full database fingerprint unchanged.",
|
||||||
|
"subject_overlap": {
|
||||||
|
"C2_to_L1": [
|
||||||
|
"coordination",
|
||||||
|
"diffusion",
|
||||||
|
"historical",
|
||||||
|
"industrial",
|
||||||
|
"internet",
|
||||||
|
"non-proliferation",
|
||||||
|
"nuclear",
|
||||||
|
"pattern-matching",
|
||||||
|
"rather",
|
||||||
|
"revolution",
|
||||||
|
"surfaced",
|
||||||
|
"technology"
|
||||||
|
],
|
||||||
|
"C3_to_L2": [
|
||||||
|
"adoption",
|
||||||
|
"better",
|
||||||
|
"causal",
|
||||||
|
"coordination",
|
||||||
|
"cost-curve",
|
||||||
|
"inflection",
|
||||||
|
"onset",
|
||||||
|
"predicts",
|
||||||
|
"technology"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"submitted_turn_audit": {
|
||||||
|
"count": 6,
|
||||||
|
"exact": true,
|
||||||
|
"expected_count": 6,
|
||||||
|
"expected_ids": [
|
||||||
|
"C1",
|
||||||
|
"L1",
|
||||||
|
"C2",
|
||||||
|
"L2",
|
||||||
|
"C3",
|
||||||
|
"L3"
|
||||||
|
],
|
||||||
|
"ids": [
|
||||||
|
"C1",
|
||||||
|
"L1",
|
||||||
|
"C2",
|
||||||
|
"L2",
|
||||||
|
"C3",
|
||||||
|
"L3"
|
||||||
|
],
|
||||||
|
"rows_are_objects": true,
|
||||||
|
"unique_id_count": 6
|
||||||
|
},
|
||||||
|
"turn_order": [
|
||||||
|
"C1",
|
||||||
|
"L1",
|
||||||
|
"C2",
|
||||||
|
"L2",
|
||||||
|
"C3",
|
||||||
|
"L3"
|
||||||
|
]
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in a new issue