fix: make challenger proof retention-safe

This commit is contained in:
twentyOne2x 2026-07-15 04:09:07 +02:00
parent 333647746f
commit b99f82bb6b
11 changed files with 479 additions and 16099 deletions

View file

@ -2,31 +2,31 @@
"wave": "wave-1", "wave": "wave-1",
"target_canary": { "target_canary": {
"working_target": "Run one isolated challenger-agent versus Leo reasoning exchange that yields distinct session identities, a transcript-visible substantive objection, a revised review-only proposal, and a no-write receipt.", "working_target": "Run one isolated challenger-agent versus Leo reasoning exchange that yields distinct session identities, a transcript-visible substantive objection, a revised review-only proposal, and a no-write receipt.",
"surface": "/private/tmp/teleo-fleet-20260715/agent-challenger-loop", "surface": "lane-owned repository worktree",
"current_tier": "T1_model_for_separate_session_claim", "current_tier": "T1_model_for_separate_session_claim",
"required_tier": "T2_runtime", "required_tier": "T2_runtime",
"first_action": "Map and reuse the existing Leo handler and trace contracts, then execute two isolated model sessions through one bounded protocol runner.", "first_action": "Map and reuse the existing Leo handler and trace contracts, then execute two isolated model sessions through one bounded protocol runner.",
"proof": "Retained transcript, per-session traces, negative-control failure, proposal review state, database fingerprints, and focused/full test readbacks." "proof": "Retained transcript, per-session traces, negative-control failure, proposal review state, database fingerprints, and focused/full test readbacks."
}, },
"ship_owner": "/root", "ship_owner": "integration_owner",
"shared_definition_of_working": { "shared_definition_of_working": {
"operator_path": "cd /private/tmp/teleo-fleet-20260715/agent-challenger-loop and run the lane-owned isolated challenger protocol command", "operator_path": "from the repository root, run the lane-owned isolated challenger protocol command",
"done_means": "T2 isolated multi-agent runtime proof, focused and relevant full tests green, and PR-shaped delivery", "done_means": "T2 isolated multi-agent runtime proof, focused and relevant full tests green, and PR-shaped delivery",
"not_done": "One model producing both sides, a scripted transcript, hidden shared memory, canonical mutation, or proof below T2", "not_done": "One model producing both sides, a scripted transcript, hidden shared memory, canonical mutation, or proof below T2",
"required_tier": "T2_runtime" "required_tier": "T2_runtime"
}, },
"shared_verifier": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/.venv/bin/python -B -m pytest -p no:cacheprovider -q tests/test_leo_agent_challenger_protocol.py tests/test_run_leo_agent_challenger_loop.py tests/test_verify_leo_unseen_reasoning_chain.py tests/test_leo_turn_execution_manifest.py tests/test_leo_tool_trace.py", "shared_verifier": "python -B -m pytest -p no:cacheprovider -q tests/test_leo_agent_challenger_protocol.py tests/test_run_leo_agent_challenger_loop.py tests/test_verify_leo_unseen_reasoning_chain.py tests/test_leo_turn_execution_manifest.py tests/test_leo_tool_trace.py",
"required_skills": [ "required_skills": [
"/Users/user/.codex/skills/subagent-wave-orchestration/SKILL.md", "subagent-wave-orchestration",
"/Users/user/.codex/skills/capability-tier-proof/SKILL.md", "capability-tier-proof",
"/Users/user/.codex/skills/codex-benchmark-lab/SKILL.md" "codex-benchmark-lab"
], ],
"must_read_before_work": true, "must_read_before_work": true,
"workers": [ "workers": [
{ {
"lane_class": "canary_dependency", "lane_class": "canary_dependency",
"planned_role": "runtime_cartographer", "planned_role": "runtime_cartographer",
"actual_agent_id_or_name": "/root/runtime_cartographer", "actual_agent_id_or_name": "runtime_cartographer",
"spawned": true, "spawned": true,
"owned_files_or_surface": ["read-only runtime, handler, and session inspection"], "owned_files_or_surface": ["read-only runtime, handler, and session inspection"],
"forbidden_files_or_surface": ["all writes", "Git index", "canonical database", "Telegram or public sends"], "forbidden_files_or_surface": ["all writes", "Git index", "canonical database", "Telegram or public sends"],
@ -39,7 +39,7 @@
{ {
"lane_class": "canary_dependency", "lane_class": "canary_dependency",
"planned_role": "negative_control_auditor", "planned_role": "negative_control_auditor",
"actual_agent_id_or_name": "/root/negative_control_auditor", "actual_agent_id_or_name": "negative_control_auditor",
"spawned": true, "spawned": true,
"owned_files_or_surface": ["read-only verifier, fixture, and proposal-contract inspection"], "owned_files_or_surface": ["read-only verifier, fixture, and proposal-contract inspection"],
"forbidden_files_or_surface": ["all writes", "Git index", "canonical database", "Telegram or public sends"], "forbidden_files_or_surface": ["all writes", "Git index", "canonical database", "Telegram or public sends"],
@ -52,7 +52,7 @@
{ {
"lane_class": "integration_owner", "lane_class": "integration_owner",
"planned_role": "delivery_runtime_auditor", "planned_role": "delivery_runtime_auditor",
"actual_agent_id_or_name": "/root/delivery_runtime_auditor", "actual_agent_id_or_name": "delivery_runtime_auditor",
"spawned": true, "spawned": true,
"owned_files_or_surface": ["read-only packaging, runner, test, CI, and delivery inspection"], "owned_files_or_surface": ["read-only packaging, runner, test, CI, and delivery inspection"],
"forbidden_files_or_surface": ["all writes", "Git index", "canonical database", "Telegram or public sends"], "forbidden_files_or_surface": ["all writes", "Git index", "canonical database", "Telegram or public sends"],
@ -65,11 +65,11 @@
{ {
"lane_class": "integration_owner", "lane_class": "integration_owner",
"planned_role": "independent_t2_judge", "planned_role": "independent_t2_judge",
"actual_agent_id_or_name": "/root/independent_t2_judge", "actual_agent_id_or_name": "independent_t2_judge",
"spawned": true, "spawned": true,
"owned_files_or_surface": ["read-only independent review of the new runner, verifier, fixtures, and focused test result"], "owned_files_or_surface": ["read-only independent review of the new runner, verifier, fixtures, and focused test result"],
"forbidden_files_or_surface": ["all writes", "Git index", "model/API calls", "canonical database", "Telegram or public sends"], "forbidden_files_or_surface": ["all writes", "Git index", "model/API calls", "canonical database", "Telegram or public sends"],
"verifier": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/.venv/bin/python -B -m pytest -p no:cacheprovider -q tests/test_leo_agent_challenger_protocol.py tests/test_run_leo_agent_challenger_loop.py tests/test_verify_leo_unseen_reasoning_chain.py tests/test_leo_turn_execution_manifest.py tests/test_leo_tool_trace.py", "verifier": "python -B -m pytest -p no:cacheprovider -q tests/test_leo_agent_challenger_protocol.py tests/test_run_leo_agent_challenger_loop.py tests/test_verify_leo_unseen_reasoning_chain.py tests/test_leo_turn_execution_manifest.py tests/test_leo_tool_trace.py",
"expected_handoff": "Independent accept or reject against T2 with any pre-live blocker", "expected_handoff": "Independent accept or reject against T2 with any pre-live blocker",
"status": "completed", "status": "completed",
"pre_live_handoff_status": "rejected_t2_closure_accepted_live_safety", "pre_live_handoff_status": "rejected_t2_closure_accepted_live_safety",
@ -118,7 +118,7 @@
"transport_attribution": "pass_from_clean_commit_6a7f2d3f8cc01cdff804a0a0d6f5c7860e385639", "transport_attribution": "pass_from_clean_commit_6a7f2d3f8cc01cdff804a0a0d6f5c7860e385639",
"challenger_behavior": "pass", "challenger_behavior": "pass",
"leo_revision_behavior_after_replay_fix": "pass", "leo_revision_behavior_after_replay_fix": "pass",
"failed_dimension": "A separate deployment advanced /opt/teleo-eval/workspaces/deploy-infra from a47415b60ef059ecc55e067ae556f49d818dc68d to 15c30f1fbe30c8f2295ddc33e293a45788a95706 during the canary, so the strict live-behavior invariance gate rejected the run." "failed_dimension": "A separate deployment advanced the deployed runtime checkout from a47415b60ef059ecc55e067ae556f49d818dc68d to 15c30f1fbe30c8f2295ddc33e293a45788a95706 during the canary, so the strict live-behavior invariance gate rejected the run."
}, },
"successful_live_canary": { "successful_live_canary": {
"remote_run_id": "7c6a2937042c", "remote_run_id": "7c6a2937042c",
@ -127,7 +127,7 @@
"verification": "pass", "verification": "pass",
"independent_judgment": "accept_no_blocking_findings", "independent_judgment": "accept_no_blocking_findings",
"source_git_commit": "af5af06130c493dd4c99c9e53638fc5b239dbee3", "source_git_commit": "af5af06130c493dd4c99c9e53638fc5b239dbee3",
"source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b", "source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e",
"recovered_remote_report_sha256": "6a7a9160095e77e5c42b7d8a2c995d3eb1e84f40624033a94ad49d616bb548a8", "recovered_remote_report_sha256": "6a7a9160095e77e5c42b7d8a2c995d3eb1e84f40624033a94ad49d616bb548a8",
"proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100", "proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100",
"database_fingerprint_sha256": "32a1f2c18f4b4c623443cbd4d9520982e9f291ec74ca63855cef51c88ac56f24", "database_fingerprint_sha256": "32a1f2c18f4b4c623443cbd4d9520982e9f291ec74ca63855cef51c88ac56f24",
@ -141,7 +141,7 @@
"canonical_mutation_performed": false "canonical_mutation_performed": false
}, },
"worktree_lifecycle": { "worktree_lifecycle": {
"contract": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/goals/leo-fleet-20260715/WORKTREE_LIFECYCLE.md", "contract": "external control copy of WORKTREE_LIFECYCLE.md (outside this repository)",
"current_finish_state": "retained_unintegrated", "current_finish_state": "retained_unintegrated",
"cleanup_owner": "integration-owner", "cleanup_owner": "integration-owner",
"worker_must_not_remove_or_prune": true "worker_must_not_remove_or_prune": true
@ -150,7 +150,7 @@
"origin_main_commit_merged": "15c30f1fbe30c8f2295ddc33e293a45788a95706", "origin_main_commit_merged": "15c30f1fbe30c8f2295ddc33e293a45788a95706",
"merge_commit": "45cf14a1df91f19b62a9235460e13c3dae616d90", "merge_commit": "45cf14a1df91f19b62a9235460e13c3dae616d90",
"feature_commits_preserved": true, "feature_commits_preserved": true,
"focused_tests": "36 passed in 0.19s", "focused_tests": "39 passed in 0.25s after integration hygiene",
"full_tests": "1297 passed in 106.35s", "full_tests": "1297 passed in 106.35s",
"ruff": "pass", "ruff": "pass",
"diff_check": "pass", "diff_check": "pass",
@ -160,6 +160,28 @@
"pr_state": "open", "pr_state": "open",
"pr_head_commit_before_lifecycle_handoff": "da84d4703eb82ebfd89971796d821250648ff985" "pr_head_commit_before_lifecycle_handoff": "da84d4703eb82ebfd89971796d821250648ff985"
}, },
"integration_hygiene": {
"repo_local_goal_file_removed": true,
"pre_projection_source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b",
"retention_safe_source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e",
"recovered_raw_remote_report_sha256": "6a7a9160095e77e5c42b7d8a2c995d3eb1e84f40624033a94ad49d616bb548a8",
"source_lines_before_after": [17330, 1402],
"behavior_inventory_entries_removed": 2658,
"receipt_and_negative_controls_regenerated": true,
"standalone_verifier": "pass_T2_runtime_zero_failed_checks_5_of_5_controls_caught",
"focused_tests": "39 passed in 0.25s",
"targeted_scan": {
"source_high_confidence_secret_matches": 0,
"source_absolute_runtime_path_matches": 0,
"source_email_matches": 0,
"source_ipv4_matches": 0,
"full_pr_delta_high_confidence_secret_matches": 0,
"full_pr_delta_local_private_path_matches": 0,
"full_pr_delta_email_matches": 0,
"full_pr_delta_ipv4_matches": 0,
"full_pr_delta_allowlisted_runtime_or_synthetic_test_path_matches": 28
}
},
"parallelism_expected": 0, "parallelism_expected": 0,
"parallelism_actual": 0, "parallelism_actual": 0,
"active_workers": [], "active_workers": [],
@ -167,5 +189,5 @@
"queued_runnable_rows": [], "queued_runnable_rows": [],
"blocked_by_tooling_or_capacity": [], "blocked_by_tooling_or_capacity": [],
"blocked_by_collision_risk": [], "blocked_by_collision_risk": [],
"repair_action": "PR 152 is open and mergeable; no repo-owned product repair remains. Keep the worktree retained_unintegrated for CI review and the integration owner's merge decision." "repair_action": "Final integration hygiene is the last repo-owned repair; keep the worktree retained_unintegrated for PR 152 and the integration owner's merge decision."
} }

View file

@ -3,3 +3,4 @@
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "partial_lower_tier", "decision": "repair", "generated_at_utc": "2026-07-15T01:14:26.233153+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "fa7f0f6e69684da6b1fb1d47660c7b5c6aa1175471bf3ed30d5f82be65fbae42", "remote_run_id": "6772a305592d", "required_tier": "T2_runtime", "source_report_sha256": "afee2f0a40f23ac830a38a26ea80f5ae8ed9be0d6459d3f19caca0acfdd4815c", "status": "fail"} {"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "partial_lower_tier", "decision": "repair", "generated_at_utc": "2026-07-15T01:14:26.233153+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "fa7f0f6e69684da6b1fb1d47660c7b5c6aa1175471bf3ed30d5f82be65fbae42", "remote_run_id": "6772a305592d", "required_tier": "T2_runtime", "source_report_sha256": "afee2f0a40f23ac830a38a26ea80f5ae8ed9be0d6459d3f19caca0acfdd4815c", "status": "fail"}
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "partial_lower_tier", "decision": "repair", "generated_at_utc": "2026-07-15T01:20:05.096585+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "b4f11109678c39bc7c6e76a1dc0ba450b31a13e4c88c2c1d28fb7ac40f77425e", "remote_run_id": "ec31613dd517", "required_tier": "T2_runtime", "source_report_sha256": "cc830da5dd5e60acd6e6fb8a54f01049cbf00a04a46b2ff7e783a6f743b1e3cf", "status": "fail"} {"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "partial_lower_tier", "decision": "repair", "generated_at_utc": "2026-07-15T01:20:05.096585+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "b4f11109678c39bc7c6e76a1dc0ba450b31a13e4c88c2c1d28fb7ac40f77425e", "remote_run_id": "ec31613dd517", "required_tier": "T2_runtime", "source_report_sha256": "cc830da5dd5e60acd6e6fb8a54f01049cbf00a04a46b2ff7e783a6f743b1e3cf", "status": "fail"}
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "T2_runtime", "decision": "keep", "generated_at_utc": "2026-07-15T01:28:36.436490+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100", "remote_run_id": "7c6a2937042c", "required_tier": "T2_runtime", "source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b", "status": "pass"} {"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "T2_runtime", "decision": "keep", "generated_at_utc": "2026-07-15T01:28:36.436490+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100", "remote_run_id": "7c6a2937042c", "required_tier": "T2_runtime", "source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b", "status": "pass"}
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "T2_runtime", "decision": "keep", "generated_at_utc": "2026-07-15T01:59:50.365865+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100", "remote_run_id": "7c6a2937042c", "required_tier": "T2_runtime", "source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e", "status": "pass"}

View file

@ -72,6 +72,6 @@
} }
], ],
"schema": "livingip.leoAgentChallengerNegativeControlReceipt.v1", "schema": "livingip.leoAgentChallengerNegativeControlReceipt.v1",
"source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b", "source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e",
"status": "pass" "status": "pass"
} }

View file

@ -28,7 +28,7 @@ The lane used the preserved project virtual environment because system Python do
- Live run: `7c6a2937042c` - Live run: `7c6a2937042c`
- Runtime source commit: `af5af06130c493dd4c99c9e53638fc5b239dbee3` - Runtime source commit: `af5af06130c493dd4c99c9e53638fc5b239dbee3`
- Source report SHA-256: `343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b` - Retention-safe source report SHA-256: `4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e`
- Recovered raw remote report SHA-256: `6a7a9160095e77e5c42b7d8a2c995d3eb1e84f40624033a94ad49d616bb548a8` - Recovered raw remote report SHA-256: `6a7a9160095e77e5c42b7d8a2c995d3eb1e84f40624033a94ad49d616bb548a8`
- Proposal packet SHA-256: `89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100` - Proposal packet SHA-256: `89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100`
- Database fingerprint SHA-256: `32a1f2c18f4b4c623443cbd4d9520982e9f291ec74ca63855cef51c88ac56f24` - Database fingerprint SHA-256: `32a1f2c18f4b4c623443cbd4d9520982e9f291ec74ca63855cef51c88ac56f24`
@ -40,6 +40,7 @@ The lane used the preserved project virtual environment because system Python do
- Post-merge focused suite: 36 passed in 0.19 seconds. - Post-merge focused suite: 36 passed in 0.19 seconds.
- Post-merge full suite: 1297 passed in 106.35 seconds. - Post-merge full suite: 1297 passed in 106.35 seconds.
- Ruff and `git diff --check`: pass. - Ruff and `git diff --check`: pass.
- Post-hygiene focused suite: 39 passed in 0.25 seconds.
Retained files: Retained files:
@ -58,7 +59,10 @@ Retained files:
- Proposal remains `pending_review`; review incomplete; apply unauthorized and unexecuted. - Proposal remains `pending_review`; review incomplete; apply unauthorized and unexecuted.
- Leo worker stopped gracefully with exit 0; no orphan worker; temporary roots removed. - Leo worker stopped gracefully with exit 0; no orphan worker; temporary roots removed.
- Initial report fetch timed out after the completed run. The exact remote file was recovered by SCP, hashed, then the - Initial report fetch timed out after the completed run. The exact remote file was recovered by SCP, hashed, then the
remote path was unlinked and verified absent. The recovery contract is retained in the source report. remote path was unlinked and verified absent. The recovery contract is retained without the private path value.
- The retained source projects both behavior manifests to their aggregate hashes, counts, and byte totals. It removes
2,658 per-file inventory rows and all absolute runtime paths while retaining the raw remote report hash and verifier
contracts. The committed source is 1,402 lines rather than the original 17,330-line transport report.
# Cost and authorization boundary # Cost and authorization boundary
@ -82,6 +86,18 @@ Retained files:
- Verify transport recovery/source attribution and the five negative controls. - Verify transport recovery/source attribution and the five negative controls.
- Verify the claim remains capped at one T2 no-send/no-write exchange. - Verify the claim remains capped at one T2 no-send/no-write exchange.
# Integration hygiene
- Removed the lane-local root `goal.md`; the durable control copy remains outside the repository.
- Regenerated the retained source, verifier receipt, negative-control receipt, and ledger entry from the accepted live
run rather than discarding the existing receipt changes.
- Reduced the retained source from 17,330 to 1,402 lines by removing 2,658 duplicated per-file behavior inventory
entries. Aggregate behavior/component hashes, counts, byte totals, raw remote report hash, and role contracts remain.
- Targeted scan of the retained source: zero high-confidence secrets, absolute runtime paths, emails, or IPv4 values.
- Targeted scan of the complete PR delta: zero high-confidence secrets, local private paths, emails, or IPv4 values.
The 28 remaining path matches are required remote runtime constants or synthetic fail-closed test fixtures.
- The standalone verifier passes at `T2_runtime` with zero failed checks and all five negative controls caught.
# Delivery and worktree lifecycle # Delivery and worktree lifecycle
- Delivery state: open PR [#152](https://github.com/living-ip/teleo-infrastructure/pull/152), mergeable at creation. - Delivery state: open PR [#152](https://github.com/living-ip/teleo-infrastructure/pull/152), mergeable at creation.

View file

@ -139,7 +139,7 @@
"f4a7aa457acc40f920b4791f2952cb8dc77148c5783e902d61f6705fbf32623d" "f4a7aa457acc40f920b4791f2952cb8dc77148c5783e902d61f6705fbf32623d"
] ]
}, },
"source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b", "source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e",
"status": "pass", "status": "pass",
"strongest_claim_allowed": "T2 isolated multi-agent runtime proof: two disjoint no-send handler processes completed a bounded challenger/Leo exchange, linked a substantive objection to a revised review-only packet, and left the full database fingerprint unchanged.", "strongest_claim_allowed": "T2 isolated multi-agent runtime proof: two disjoint no-send handler processes completed a bounded challenger/Leo exchange, linked a substantive objection to a revised review-only packet, and left the full database fingerprint unchanged.",
"subject_overlap": { "subject_overlap": {

46
goal.md
View file

@ -1,46 +0,0 @@
# Agent-As-User Challenger Loop Goal
## Working target
Prove that a separate challenger agent can act as the user, interrogate Leo's claim and evidence, object to an overreach, and drive a revised review-only proposal without hidden shared memory or canonical mutation.
## Operator path
Start isolated Leo and challenger sessions, give the challenger only the user objective and transcript-visible context, run a bounded multi-turn protocol, and independently verify tool usage, proposal state, and database invariance.
## Done means
- Challenger and Leo have distinct session identities and no hidden shared state.
- The challenger asks a natural initial question, follows evidence weaknesses, and raises one substantive objection.
- Leo revises the candidate rather than defending or silently applying it.
- The resulting proposal packet is reviewable and replayable.
- At least one negative control catches collusion, leaked row IDs, or unsupported agreement.
## Not done
One model generating both sides in one prompt, a scripted transcript, or two agents sharing unrestricted memory is not agent-to-agent proof.
## Required tier
T2 isolated multi-agent runtime proof; T3 handler execution is the next tier after local closure.
## Current truth
The human-style three-turn chain works in one handler session. A live separated agent-as-user version has not been proven.
## Owned surface
New challenger protocol runner, isolated session fixtures, verifier, and tests. Reuse benchmark contracts without editing the OOS scorer or Telegram sender.
## Forbidden
No canonical apply, no hidden transcript injection, no shared writable memory, no public send, and no claim that simulated agents equal human acceptance.
## Required skills
- `/Users/user/.codex/skills/subagent-wave-orchestration/SKILL.md`
- `/Users/user/.codex/skills/capability-tier-proof/SKILL.md`
- `/Users/user/.codex/skills/codex-benchmark-lab/SKILL.md`
## First action
Create the platform Goal, define the session isolation contract and one minimal challenger/Leo canary, then run it with full traces before adding complexity.
## Delivery and proof
Deliver a PR with the protocol, negative controls, retained transcript/trace receipt, tests, and explicit limitations.
## Platform Goal objective
Complete and verify the isolated agent-as-user challenge and revision loop defined in this goal file.
## Worktree lifecycle
At completion, report the delivery disposition and exact dirty state. Follow `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/goals/leo-fleet-20260715/WORKTREE_LIFECYCLE.md`. Keep the worktree while any useful commit, PR, patch, proof, or dirty delta remains unintegrated. The integration owner removes and prunes it only after merge, deliberate port, or explicit rejection is verified.

View file

@ -669,6 +669,10 @@ def verify_report(
and roles.get("distinct_profile_roots") is True and roles.get("distinct_profile_roots") is True
and challenger_role.get("process_identity_sha256") and challenger_role.get("process_identity_sha256")
!= leo_role.get("process_identity_sha256") != leo_role.get("process_identity_sha256")
and challenger_role.get("profile_realpath_sha256")
and leo_role.get("profile_realpath_sha256")
and challenger_role.get("profile_realpath_sha256")
!= leo_role.get("profile_realpath_sha256")
and challenger_role.get("contract_sha256") and challenger_role.get("contract_sha256")
== compute_runtime_role_contract_sha256(challenger_role) == compute_runtime_role_contract_sha256(challenger_role)
and leo_role.get("contract_sha256") == compute_runtime_role_contract_sha256(leo_role) and leo_role.get("contract_sha256") == compute_runtime_role_contract_sha256(leo_role)

View file

@ -4,6 +4,7 @@
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import copy
import hashlib import hashlib
import json import json
import re import re
@ -29,6 +30,25 @@ LEDGER_JSONL = REPORT_DIR / "leo-agent-challenger-loop-ledger.jsonl"
DEFAULT_MODEL = "google/gemini-2.5-flash" DEFAULT_MODEL = "google/gemini-2.5-flash"
DEFAULT_MAX_TOKENS = 768 DEFAULT_MAX_TOKENS = 768
DEFAULT_REPORT_PREFIX = "leo-agent-challenger-loop" DEFAULT_REPORT_PREFIX = "leo-agent-challenger-loop"
RETENTION_PROJECTION_SCHEMA = "livingip.leoAgentChallengerRetentionProjection.v1"
BEHAVIOR_RETENTION_SCHEMA = "livingip.leoBehaviorManifestRetentionProjection.v1"
RETENTION_FORBIDDEN_PATTERNS = (
(
"absolute runtime path",
re.compile(r"(?<![A-Za-z0-9])/(?:Users|private|home|root|opt|tmp|var|etc)/[^\s\"']+"),
),
("PEM private key", re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")),
("credentialed URL", re.compile(r"https?://[^/\s:@]+:[^/\s@]+@")),
("JWT", re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b")),
(
"known token form",
re.compile(
r"\b(?:sk-[A-Za-z0-9_-]{20,}|AIza[0-9A-Za-z_-]{35}|gh[pousr]_[A-Za-z0-9]{20,}|"
r"xox[baprs]-[A-Za-z0-9-]{20,})\b"
),
),
("Bearer credential", re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]{20,}\b", re.IGNORECASE)),
)
LEO_CHALLENGER_SOUL_OVERLAY = """ LEO_CHALLENGER_SOUL_OVERLAY = """
@ -1277,6 +1297,155 @@ def run_remote(
return result return result
def _sha256_text(value: Any) -> str:
return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest()
def _manifest_content_projection(content: Any) -> dict[str, Any]:
if not isinstance(content, dict):
return {}
return {
"file_count": content.get("file_count"),
"total_bytes": content.get("total_bytes"),
"sha256": content.get("sha256"),
"missing_count": len(content.get("missing") or []),
"symlink_count": len(content.get("symlinks") or []),
}
def compact_behavior_manifest_for_retention(manifest: Any) -> dict[str, Any]:
"""Keep behavior equality proof without retaining private per-file inventory."""
if not isinstance(manifest, dict):
return {}
projection = manifest.get("retention_projection")
if isinstance(projection, dict) and projection.get("schema") == BEHAVIOR_RETENTION_SCHEMA:
return copy.deepcopy(manifest)
components: dict[str, Any] = {}
removed_file_entries = 0
removed_missing_entries = 0
removed_symlink_entries = 0
raw_components = manifest.get("components") if isinstance(manifest.get("components"), dict) else {}
for name, raw_component in sorted(raw_components.items()):
component = raw_component if isinstance(raw_component, dict) else {}
content = component.get("content") if isinstance(component.get("content"), dict) else {}
removed_file_entries += len(content.get("files") or [])
removed_missing_entries += len(content.get("missing") or [])
removed_symlink_entries += len(content.get("symlinks") or [])
components[str(name)] = {
key: copy.deepcopy(component.get(key))
for key in ("role", "mutability", "replayability")
}
components[str(name)]["content"] = _manifest_content_projection(content)
runtimes: dict[str, Any] = {}
for name in ("hermes_runtime", "teleo_infrastructure_runtime"):
runtime = manifest.get(name) if isinstance(manifest.get(name), dict) else {}
source_tree = runtime.get("source_tree") if isinstance(runtime.get("source_tree"), dict) else {}
removed_file_entries += len(source_tree.get("files") or [])
removed_missing_entries += len(source_tree.get("missing") or [])
runtimes[name] = {
"git_head": runtime.get("git_head"),
"git_state": copy.deepcopy(runtime.get("git_state")),
"source_tree": _manifest_content_projection(source_tree),
}
removed_root_count = sum(
1 for name in ("profile_root", "hermes_root", "deployment_root") if manifest.get(name)
)
return {
"schema": manifest.get("schema"),
"generated_at_utc": manifest.get("generated_at_utc"),
"behavior_sha256": manifest.get("behavior_sha256"),
"model_runtime": copy.deepcopy(manifest.get("model_runtime")),
"canonical_database": copy.deepcopy(manifest.get("canonical_database")),
"components": components,
**runtimes,
"retention_projection": {
"schema": BEHAVIOR_RETENTION_SCHEMA,
"raw_manifest_sha256": protocol.canonical_sha256(manifest),
"absolute_root_values_removed": True,
"absolute_root_values_removed_count": removed_root_count,
"per_file_inventories_removed": True,
"removed_file_entries": removed_file_entries,
"removed_missing_entries": removed_missing_entries,
"removed_symlink_entries": removed_symlink_entries,
},
}
def sanitize_report_for_retention(report: dict[str, Any]) -> dict[str, Any]:
"""Project a live report into a review-safe, independently verifiable receipt."""
retained = copy.deepcopy(report)
projection = retained.get("retention_projection")
if isinstance(projection, dict) and projection.get("schema") == RETENTION_PROJECTION_SCHEMA:
assert_report_safe_for_retention(retained)
return retained
pre_projection_report_sha256 = protocol.canonical_sha256(retained)
behavior_projection: dict[str, Any] = {}
for field in ("live_behavior_manifest_before", "live_behavior_manifest_after"):
raw_manifest = retained.get(field)
compacted = compact_behavior_manifest_for_retention(raw_manifest)
retained[field] = compacted
behavior_projection[field] = copy.deepcopy(compacted.get("retention_projection"))
redacted_fields: list[str] = []
remote_report_path = retained.pop("remote_report_path", None)
if remote_report_path:
redacted_fields.append("remote_report_path")
transport = retained.get("execution_transport")
if isinstance(transport, dict) and isinstance(transport.get("recovery"), dict):
recovery = transport["recovery"]
remote_path = recovery.pop("remote_path", None)
if remote_path:
redacted_fields.append("execution_transport.recovery.remote_path")
original_role_contracts: dict[str, str] = {}
profile_realpath_hash_verified: dict[str, bool] = {}
isolation = retained.get("isolation") if isinstance(retained.get("isolation"), dict) else {}
roles = isolation.get("roles") if isinstance(isolation.get("roles"), dict) else {}
for name, role in sorted(roles.items()):
if not isinstance(role, dict):
continue
original_role_contracts[str(name)] = str(role.get("contract_sha256") or "")
profile_realpath = role.get("profile_realpath")
if profile_realpath:
matches = _sha256_text(profile_realpath) == role.get("profile_realpath_sha256")
profile_realpath_hash_verified[str(name)] = matches
if not matches:
raise ValueError(f"{name} profile realpath hash does not match before retention")
redacted_fields.append(f"isolation.roles.{name}.profile_realpath")
role["profile_realpath"] = None
role["contract_sha256"] = protocol.compute_runtime_role_contract_sha256(role)
recovery = transport.get("recovery") if isinstance(transport, dict) else {}
retained["retention_projection"] = {
"schema": RETENTION_PROJECTION_SCHEMA,
"raw_remote_report_sha256": (
recovery.get("recovered_report_sha256") if isinstance(recovery, dict) else None
),
"pre_projection_report_sha256": pre_projection_report_sha256,
"behavior_manifests": behavior_projection,
"redacted_fields": sorted(redacted_fields),
"redacted_field_count": len(redacted_fields),
"profile_realpath_hash_verified": profile_realpath_hash_verified,
"original_runtime_role_contract_sha256": original_role_contracts,
"absolute_runtime_paths_removed": True,
"verifier_contract_recomputed_after_redaction": True,
}
assert_report_safe_for_retention(retained)
return retained
def assert_report_safe_for_retention(report: dict[str, Any]) -> None:
encoded = json.dumps(report, sort_keys=True, ensure_ascii=True)
for label, pattern in RETENTION_FORBIDDEN_PATTERNS:
if pattern.search(encoded):
raise ValueError(f"retained report contains forbidden {label}")
def write_outputs(remote: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: def write_outputs(remote: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
REPORT_DIR.mkdir(parents=True, exist_ok=True) REPORT_DIR.mkdir(parents=True, exist_ok=True)
report = remote.get("parsed") if isinstance(remote.get("parsed"), dict) else {} report = remote.get("parsed") if isinstance(remote.get("parsed"), dict) else {}
@ -1285,6 +1454,7 @@ def write_outputs(remote: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any
report["execution_transport"] = remote.get("execution_transport") report["execution_transport"] = remote.get("execution_transport")
if remote.get("stderr"): if remote.get("stderr"):
report["remote_stderr"] = remote["stderr"] report["remote_stderr"] = remote["stderr"]
report = sanitize_report_for_retention(report)
OUTPUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") OUTPUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
source_sha = hashlib.sha256(OUTPUT_JSON.read_bytes()).hexdigest() source_sha = hashlib.sha256(OUTPUT_JSON.read_bytes()).hexdigest()
receipt = protocol.verify_report(report, source_report_sha256=source_sha) receipt = protocol.verify_report(report, source_report_sha256=source_sha)

View file

@ -373,6 +373,18 @@ def test_same_handler_or_model_session_fails_separation() -> None:
assert receipt["checks"]["distinct_model_session_identities"] is False assert receipt["checks"]["distinct_model_session_identities"] is False
def test_same_profile_root_hash_fails_separation_without_requiring_raw_paths() -> None:
report = passing_report()
challenger = report["isolation"]["roles"]["challenger"]
leo = report["isolation"]["roles"]["leo"]
leo["profile_realpath_sha256"] = challenger["profile_realpath_sha256"]
leo["contract_sha256"] = protocol.compute_runtime_role_contract_sha256(leo)
receipt = protocol.verify_report(report, source_report_sha256="4" * 64)
assert receipt["checks"]["separate_process_and_profile_roots"] is False
def test_repeated_execution_hashes_fail_even_with_valid_parent_transcript() -> None: def test_repeated_execution_hashes_fail_even_with_valid_parent_transcript() -> None:
report = passing_report() report = passing_report()
for turn in report["turns"]: for turn in report["turns"]:

View file

@ -1,5 +1,7 @@
from __future__ import annotations from __future__ import annotations
import json
import pytest import pytest
from scripts import run_leo_agent_challenger_loop as runner from scripts import run_leo_agent_challenger_loop as runner
@ -30,3 +32,130 @@ def test_remote_script_rejects_unbounded_tokens_and_unsafe_report_prefix() -> No
with pytest.raises(ValueError, match="report_prefix"): with pytest.raises(ValueError, match="report_prefix"):
runner.build_remote_script("abc123", report_prefix="bad/path") runner.build_remote_script("abc123", report_prefix="bad/path")
def test_retention_projection_removes_runtime_paths_without_losing_hash_proof() -> None:
manifest = {
"schema": "livingip.leoBehaviorManifest.v1",
"generated_at_utc": "2026-07-15T00:00:00+00:00",
"behavior_sha256": "a" * 64,
"profile_root": "/home/teleo/.hermes/profiles/leoclean",
"hermes_root": "/home/teleo/.hermes/hermes-agent",
"deployment_root": "/opt/teleo-eval/workspaces/deploy-infra",
"model_runtime": {"status": "observed"},
"canonical_database": {"included": False},
"components": {
"conversation_sessions": {
"role": "session state",
"mutability": "dynamic",
"replayability": "hashable",
"content": {
"files": [{"path": "/home/teleo/private/session.json", "sha256": "b" * 64}],
"missing": [],
"symlinks": [],
"file_count": 1,
"total_bytes": 42,
"sha256": "c" * 64,
},
}
},
"hermes_runtime": {
"git_head": "d" * 40,
"git_state": {"worktree_clean": True},
"source_tree": {
"files": [{"path": "/home/teleo/private/runtime.py", "sha256": "e" * 64}],
"missing": [],
"file_count": 1,
"total_bytes": 84,
"sha256": "f" * 64,
},
},
"teleo_infrastructure_runtime": {
"git_head": "1" * 40,
"git_state": {"worktree_clean": True},
"source_tree": {
"files": [{"path": "/opt/teleo-eval/private/runner.py", "sha256": "2" * 64}],
"missing": [],
"file_count": 1,
"total_bytes": 126,
"sha256": "3" * 64,
},
},
}
role = {
"role": "leo",
"runtime_kind": "gatewayrunner_temp_profile_no_model_tools",
"profile_realpath": "/tmp/leo-agent-challenger-loop-test/leo-profile",
}
role["profile_realpath_sha256"] = runner._sha256_text(role["profile_realpath"])
role["contract_sha256"] = runner.protocol.compute_runtime_role_contract_sha256(role)
original_contract = role["contract_sha256"]
raw = {
"remote_report_path": "/tmp/leo-agent-challenger-loop-test.json",
"execution_transport": {
"recovery": {
"remote_path": "/tmp/leo-agent-challenger-loop-test.json",
"recovered_report_sha256": "5" * 64,
}
},
"isolation": {"roles": {"leo": role}},
"live_behavior_manifest_before": manifest,
"live_behavior_manifest_after": manifest,
}
retained = runner.sanitize_report_for_retention(raw)
encoded = json.dumps(retained, sort_keys=True)
assert "/home/" not in encoded
assert "/opt/" not in encoded
assert "/tmp/" not in encoded
assert raw["remote_report_path"].startswith("/tmp/")
assert retained["live_behavior_manifest_before"]["components"]["conversation_sessions"][
"content"
] == {
"file_count": 1,
"missing_count": 0,
"sha256": "c" * 64,
"symlink_count": 0,
"total_bytes": 42,
}
assert retained["live_behavior_manifest_before"]["retention_projection"][
"removed_file_entries"
] == 3
retained_role = retained["isolation"]["roles"]["leo"]
assert retained_role["profile_realpath"] is None
assert retained_role["profile_realpath_sha256"] == runner._sha256_text(
"/tmp/leo-agent-challenger-loop-test/leo-profile"
)
assert retained_role["contract_sha256"] == runner.protocol.compute_runtime_role_contract_sha256(
retained_role
)
assert retained["retention_projection"]["original_runtime_role_contract_sha256"]["leo"] == (
original_contract
)
assert retained["retention_projection"]["raw_remote_report_sha256"] == "5" * 64
assert retained["retention_projection"]["profile_realpath_hash_verified"] == {"leo": True}
assert runner.sanitize_report_for_retention(retained) == retained
retained["remote_stderr"] = "traceback from /home/teleo/private/runtime.py"
with pytest.raises(ValueError, match="forbidden absolute runtime path"):
runner.sanitize_report_for_retention(retained)
def test_retention_projection_fails_closed_on_mismatched_path_hash_or_unexpected_path() -> None:
mismatched_role = {
"role": "leo",
"profile_realpath": "/tmp/leo-profile",
"profile_realpath_sha256": "0" * 64,
}
mismatched_role["contract_sha256"] = runner.protocol.compute_runtime_role_contract_sha256(
mismatched_role
)
with pytest.raises(ValueError, match="profile realpath hash does not match"):
runner.sanitize_report_for_retention(
{"isolation": {"roles": {"leo": mismatched_role}}}
)
with pytest.raises(ValueError, match="forbidden absolute runtime path"):
runner.sanitize_report_for_retention(
{"remote_stderr": "traceback from /home/teleo/private/runtime.py"}
)