fix: make challenger proof retention-safe
This commit is contained in:
twentyOne2x
parent
333647746f
commit
b99f82bb6b
11 changed files with 479 additions and 16099 deletions
|
|
@ -2,31 +2,31 @@
|
|||
"wave": "wave-1",
|
||||
"target_canary": {
|
||||
"working_target": "Run one isolated challenger-agent versus Leo reasoning exchange that yields distinct session identities, a transcript-visible substantive objection, a revised review-only proposal, and a no-write receipt.",
|
||||
"surface": "/private/tmp/teleo-fleet-20260715/agent-challenger-loop",
|
||||
"surface": "lane-owned repository worktree",
|
||||
"current_tier": "T1_model_for_separate_session_claim",
|
||||
"required_tier": "T2_runtime",
|
||||
"first_action": "Map and reuse the existing Leo handler and trace contracts, then execute two isolated model sessions through one bounded protocol runner.",
|
||||
"proof": "Retained transcript, per-session traces, negative-control failure, proposal review state, database fingerprints, and focused/full test readbacks."
|
||||
},
|
||||
"ship_owner": "/root",
|
||||
"ship_owner": "integration_owner",
|
||||
"shared_definition_of_working": {
|
||||
"operator_path": "cd /private/tmp/teleo-fleet-20260715/agent-challenger-loop and run the lane-owned isolated challenger protocol command",
|
||||
"operator_path": "from the repository root, run the lane-owned isolated challenger protocol command",
|
||||
"done_means": "T2 isolated multi-agent runtime proof, focused and relevant full tests green, and PR-shaped delivery",
|
||||
"not_done": "One model producing both sides, a scripted transcript, hidden shared memory, canonical mutation, or proof below T2",
|
||||
"required_tier": "T2_runtime"
|
||||
},
|
||||
"shared_verifier": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/.venv/bin/python -B -m pytest -p no:cacheprovider -q tests/test_leo_agent_challenger_protocol.py tests/test_run_leo_agent_challenger_loop.py tests/test_verify_leo_unseen_reasoning_chain.py tests/test_leo_turn_execution_manifest.py tests/test_leo_tool_trace.py",
|
||||
"shared_verifier": "python -B -m pytest -p no:cacheprovider -q tests/test_leo_agent_challenger_protocol.py tests/test_run_leo_agent_challenger_loop.py tests/test_verify_leo_unseen_reasoning_chain.py tests/test_leo_turn_execution_manifest.py tests/test_leo_tool_trace.py",
|
||||
"required_skills": [
|
||||
"/Users/user/.codex/skills/subagent-wave-orchestration/SKILL.md",
|
||||
"/Users/user/.codex/skills/capability-tier-proof/SKILL.md",
|
||||
"/Users/user/.codex/skills/codex-benchmark-lab/SKILL.md"
|
||||
"subagent-wave-orchestration",
|
||||
"capability-tier-proof",
|
||||
"codex-benchmark-lab"
|
||||
],
|
||||
"must_read_before_work": true,
|
||||
"workers": [
|
||||
{
|
||||
"lane_class": "canary_dependency",
|
||||
"planned_role": "runtime_cartographer",
|
||||
"actual_agent_id_or_name": "/root/runtime_cartographer",
|
||||
"actual_agent_id_or_name": "runtime_cartographer",
|
||||
"spawned": true,
|
||||
"owned_files_or_surface": ["read-only runtime, handler, and session inspection"],
|
||||
"forbidden_files_or_surface": ["all writes", "Git index", "canonical database", "Telegram or public sends"],
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
{
|
||||
"lane_class": "canary_dependency",
|
||||
"planned_role": "negative_control_auditor",
|
||||
"actual_agent_id_or_name": "/root/negative_control_auditor",
|
||||
"actual_agent_id_or_name": "negative_control_auditor",
|
||||
"spawned": true,
|
||||
"owned_files_or_surface": ["read-only verifier, fixture, and proposal-contract inspection"],
|
||||
"forbidden_files_or_surface": ["all writes", "Git index", "canonical database", "Telegram or public sends"],
|
||||
|
|
@ -52,7 +52,7 @@
|
|||
{
|
||||
"lane_class": "integration_owner",
|
||||
"planned_role": "delivery_runtime_auditor",
|
||||
"actual_agent_id_or_name": "/root/delivery_runtime_auditor",
|
||||
"actual_agent_id_or_name": "delivery_runtime_auditor",
|
||||
"spawned": true,
|
||||
"owned_files_or_surface": ["read-only packaging, runner, test, CI, and delivery inspection"],
|
||||
"forbidden_files_or_surface": ["all writes", "Git index", "canonical database", "Telegram or public sends"],
|
||||
|
|
@ -65,11 +65,11 @@
|
|||
{
|
||||
"lane_class": "integration_owner",
|
||||
"planned_role": "independent_t2_judge",
|
||||
"actual_agent_id_or_name": "/root/independent_t2_judge",
|
||||
"actual_agent_id_or_name": "independent_t2_judge",
|
||||
"spawned": true,
|
||||
"owned_files_or_surface": ["read-only independent review of the new runner, verifier, fixtures, and focused test result"],
|
||||
"forbidden_files_or_surface": ["all writes", "Git index", "model/API calls", "canonical database", "Telegram or public sends"],
|
||||
"verifier": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/.venv/bin/python -B -m pytest -p no:cacheprovider -q tests/test_leo_agent_challenger_protocol.py tests/test_run_leo_agent_challenger_loop.py tests/test_verify_leo_unseen_reasoning_chain.py tests/test_leo_turn_execution_manifest.py tests/test_leo_tool_trace.py",
|
||||
"verifier": "python -B -m pytest -p no:cacheprovider -q tests/test_leo_agent_challenger_protocol.py tests/test_run_leo_agent_challenger_loop.py tests/test_verify_leo_unseen_reasoning_chain.py tests/test_leo_turn_execution_manifest.py tests/test_leo_tool_trace.py",
|
||||
"expected_handoff": "Independent accept or reject against T2 with any pre-live blocker",
|
||||
"status": "completed",
|
||||
"pre_live_handoff_status": "rejected_t2_closure_accepted_live_safety",
|
||||
|
|
@ -118,7 +118,7 @@
|
|||
"transport_attribution": "pass_from_clean_commit_6a7f2d3f8cc01cdff804a0a0d6f5c7860e385639",
|
||||
"challenger_behavior": "pass",
|
||||
"leo_revision_behavior_after_replay_fix": "pass",
|
||||
"failed_dimension": "A separate deployment advanced /opt/teleo-eval/workspaces/deploy-infra from a47415b60ef059ecc55e067ae556f49d818dc68d to 15c30f1fbe30c8f2295ddc33e293a45788a95706 during the canary, so the strict live-behavior invariance gate rejected the run."
|
||||
"failed_dimension": "A separate deployment advanced the deployed runtime checkout from a47415b60ef059ecc55e067ae556f49d818dc68d to 15c30f1fbe30c8f2295ddc33e293a45788a95706 during the canary, so the strict live-behavior invariance gate rejected the run."
|
||||
},
|
||||
"successful_live_canary": {
|
||||
"remote_run_id": "7c6a2937042c",
|
||||
|
|
@ -127,7 +127,7 @@
|
|||
"verification": "pass",
|
||||
"independent_judgment": "accept_no_blocking_findings",
|
||||
"source_git_commit": "af5af06130c493dd4c99c9e53638fc5b239dbee3",
|
||||
"source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b",
|
||||
"source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e",
|
||||
"recovered_remote_report_sha256": "6a7a9160095e77e5c42b7d8a2c995d3eb1e84f40624033a94ad49d616bb548a8",
|
||||
"proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100",
|
||||
"database_fingerprint_sha256": "32a1f2c18f4b4c623443cbd4d9520982e9f291ec74ca63855cef51c88ac56f24",
|
||||
|
|
@ -141,7 +141,7 @@
|
|||
"canonical_mutation_performed": false
|
||||
},
|
||||
"worktree_lifecycle": {
|
||||
"contract": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/goals/leo-fleet-20260715/WORKTREE_LIFECYCLE.md",
|
||||
"contract": "external control copy of WORKTREE_LIFECYCLE.md (outside this repository)",
|
||||
"current_finish_state": "retained_unintegrated",
|
||||
"cleanup_owner": "integration-owner",
|
||||
"worker_must_not_remove_or_prune": true
|
||||
|
|
@ -150,7 +150,7 @@
|
|||
"origin_main_commit_merged": "15c30f1fbe30c8f2295ddc33e293a45788a95706",
|
||||
"merge_commit": "45cf14a1df91f19b62a9235460e13c3dae616d90",
|
||||
"feature_commits_preserved": true,
|
||||
"focused_tests": "36 passed in 0.19s",
|
||||
"focused_tests": "39 passed in 0.25s after integration hygiene",
|
||||
"full_tests": "1297 passed in 106.35s",
|
||||
"ruff": "pass",
|
||||
"diff_check": "pass",
|
||||
|
|
@ -160,6 +160,28 @@
|
|||
"pr_state": "open",
|
||||
"pr_head_commit_before_lifecycle_handoff": "da84d4703eb82ebfd89971796d821250648ff985"
|
||||
},
|
||||
"integration_hygiene": {
|
||||
"repo_local_goal_file_removed": true,
|
||||
"pre_projection_source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b",
|
||||
"retention_safe_source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e",
|
||||
"recovered_raw_remote_report_sha256": "6a7a9160095e77e5c42b7d8a2c995d3eb1e84f40624033a94ad49d616bb548a8",
|
||||
"source_lines_before_after": [17330, 1402],
|
||||
"behavior_inventory_entries_removed": 2658,
|
||||
"receipt_and_negative_controls_regenerated": true,
|
||||
"standalone_verifier": "pass_T2_runtime_zero_failed_checks_5_of_5_controls_caught",
|
||||
"focused_tests": "39 passed in 0.25s",
|
||||
"targeted_scan": {
|
||||
"source_high_confidence_secret_matches": 0,
|
||||
"source_absolute_runtime_path_matches": 0,
|
||||
"source_email_matches": 0,
|
||||
"source_ipv4_matches": 0,
|
||||
"full_pr_delta_high_confidence_secret_matches": 0,
|
||||
"full_pr_delta_local_private_path_matches": 0,
|
||||
"full_pr_delta_email_matches": 0,
|
||||
"full_pr_delta_ipv4_matches": 0,
|
||||
"full_pr_delta_allowlisted_runtime_or_synthetic_test_path_matches": 28
|
||||
}
|
||||
},
|
||||
"parallelism_expected": 0,
|
||||
"parallelism_actual": 0,
|
||||
"active_workers": [],
|
||||
|
|
@ -167,5 +189,5 @@
|
|||
"queued_runnable_rows": [],
|
||||
"blocked_by_tooling_or_capacity": [],
|
||||
"blocked_by_collision_risk": [],
|
||||
"repair_action": "PR 152 is open and mergeable; no repo-owned product repair remains. Keep the worktree retained_unintegrated for CI review and the integration owner's merge decision."
|
||||
"repair_action": "Final integration hygiene is the last repo-owned repair; keep the worktree retained_unintegrated for PR 152 and the integration owner's merge decision."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,3 +3,4 @@
|
|||
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "partial_lower_tier", "decision": "repair", "generated_at_utc": "2026-07-15T01:14:26.233153+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "fa7f0f6e69684da6b1fb1d47660c7b5c6aa1175471bf3ed30d5f82be65fbae42", "remote_run_id": "6772a305592d", "required_tier": "T2_runtime", "source_report_sha256": "afee2f0a40f23ac830a38a26ea80f5ae8ed9be0d6459d3f19caca0acfdd4815c", "status": "fail"}
|
||||
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "partial_lower_tier", "decision": "repair", "generated_at_utc": "2026-07-15T01:20:05.096585+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "b4f11109678c39bc7c6e76a1dc0ba450b31a13e4c88c2c1d28fb7ac40f77425e", "remote_run_id": "ec31613dd517", "required_tier": "T2_runtime", "source_report_sha256": "cc830da5dd5e60acd6e6fb8a54f01049cbf00a04a46b2ff7e783a6f743b1e3cf", "status": "fail"}
|
||||
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "T2_runtime", "decision": "keep", "generated_at_utc": "2026-07-15T01:28:36.436490+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100", "remote_run_id": "7c6a2937042c", "required_tier": "T2_runtime", "source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b", "status": "pass"}
|
||||
{"benchmark_id": "leo-agent-challenger-loop-20260715", "current_tier": "T2_runtime", "decision": "keep", "generated_at_utc": "2026-07-15T01:59:50.365865+00:00", "negative_controls": ["caught", "caught", "caught", "caught", "caught"], "proposal_packet_sha256": "89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100", "remote_run_id": "7c6a2937042c", "required_tier": "T2_runtime", "source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e", "status": "pass"}
|
||||
|
|
|
|||
|
|
@ -72,6 +72,6 @@
|
|||
}
|
||||
],
|
||||
"schema": "livingip.leoAgentChallengerNegativeControlReceipt.v1",
|
||||
"source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b",
|
||||
"source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e",
|
||||
"status": "pass"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ The lane used the preserved project virtual environment because system Python do
|
|||
|
||||
- Live run: `7c6a2937042c`
|
||||
- Runtime source commit: `af5af06130c493dd4c99c9e53638fc5b239dbee3`
|
||||
- Source report SHA-256: `343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b`
|
||||
- Retention-safe source report SHA-256: `4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e`
|
||||
- Recovered raw remote report SHA-256: `6a7a9160095e77e5c42b7d8a2c995d3eb1e84f40624033a94ad49d616bb548a8`
|
||||
- Proposal packet SHA-256: `89c543b8a12e922044f93bcc6b1e23a3a252b044dc1d45163575b70bc1a3c100`
|
||||
- Database fingerprint SHA-256: `32a1f2c18f4b4c623443cbd4d9520982e9f291ec74ca63855cef51c88ac56f24`
|
||||
|
|
@ -40,6 +40,7 @@ The lane used the preserved project virtual environment because system Python do
|
|||
- Post-merge focused suite: 36 passed in 0.19 seconds.
|
||||
- Post-merge full suite: 1297 passed in 106.35 seconds.
|
||||
- Ruff and `git diff --check`: pass.
|
||||
- Post-hygiene focused suite: 39 passed in 0.25 seconds.
|
||||
|
||||
Retained files:
|
||||
|
||||
|
|
@ -58,7 +59,10 @@ Retained files:
|
|||
- Proposal remains `pending_review`; review incomplete; apply unauthorized and unexecuted.
|
||||
- Leo worker stopped gracefully with exit 0; no orphan worker; temporary roots removed.
|
||||
- Initial report fetch timed out after the completed run. The exact remote file was recovered by SCP, hashed, then the
|
||||
remote path was unlinked and verified absent. The recovery contract is retained in the source report.
|
||||
remote path was unlinked and verified absent. The recovery contract is retained without the private path value.
|
||||
- The retained source projects both behavior manifests to their aggregate hashes, counts, and byte totals. It removes
|
||||
2,658 per-file inventory rows and all absolute runtime paths while retaining the raw remote report hash and verifier
|
||||
contracts. The committed source is 1,402 lines rather than the original 17,330-line transport report.
|
||||
|
||||
# Cost and authorization boundary
|
||||
|
||||
|
|
@ -82,6 +86,18 @@ Retained files:
|
|||
- Verify transport recovery/source attribution and the five negative controls.
|
||||
- Verify the claim remains capped at one T2 no-send/no-write exchange.
|
||||
|
||||
# Integration hygiene
|
||||
|
||||
- Removed the lane-local root `goal.md`; the durable control copy remains outside the repository.
|
||||
- Regenerated the retained source, verifier receipt, negative-control receipt, and ledger entry from the accepted live
|
||||
run rather than discarding the existing receipt changes.
|
||||
- Reduced the retained source from 17,330 to 1,402 lines by removing 2,658 duplicated per-file behavior inventory
|
||||
entries. Aggregate behavior/component hashes, counts, byte totals, raw remote report hash, and role contracts remain.
|
||||
- Targeted scan of the retained source: zero high-confidence secrets, absolute runtime paths, emails, or IPv4 values.
|
||||
- Targeted scan of the complete PR delta: zero high-confidence secrets, local private paths, emails, or IPv4 values.
|
||||
The 28 remaining path matches are required remote runtime constants or synthetic fail-closed test fixtures.
|
||||
- The standalone verifier passes at `T2_runtime` with zero failed checks and all five negative controls caught.
|
||||
|
||||
# Delivery and worktree lifecycle
|
||||
|
||||
- Delivery state: open PR [#152](https://github.com/living-ip/teleo-infrastructure/pull/152), mergeable at creation.
|
||||
|
|
|
|||
|
|
@ -139,7 +139,7 @@
|
|||
"f4a7aa457acc40f920b4791f2952cb8dc77148c5783e902d61f6705fbf32623d"
|
||||
]
|
||||
},
|
||||
"source_report_sha256": "343f18688334d5166bbb951f8672ccf4ff5a3a2b14e05f5279d3e1165b76f83b",
|
||||
"source_report_sha256": "4fd4269b795c32629d4b250cb66d56ceadaa93eaaa1f8f00a5ec19d76bf4f09e",
|
||||
"status": "pass",
|
||||
"strongest_claim_allowed": "T2 isolated multi-agent runtime proof: two disjoint no-send handler processes completed a bounded challenger/Leo exchange, linked a substantive objection to a revised review-only packet, and left the full database fingerprint unchanged.",
|
||||
"subject_overlap": {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
46
goal.md
46
goal.md
|
|
@ -1,46 +0,0 @@
|
|||
# Agent-As-User Challenger Loop Goal
|
||||
|
||||
## Working target
|
||||
Prove that a separate challenger agent can act as the user, interrogate Leo's claim and evidence, object to an overreach, and drive a revised review-only proposal without hidden shared memory or canonical mutation.
|
||||
|
||||
## Operator path
|
||||
Start isolated Leo and challenger sessions, give the challenger only the user objective and transcript-visible context, run a bounded multi-turn protocol, and independently verify tool usage, proposal state, and database invariance.
|
||||
|
||||
## Done means
|
||||
- Challenger and Leo have distinct session identities and no hidden shared state.
|
||||
- The challenger asks a natural initial question, follows evidence weaknesses, and raises one substantive objection.
|
||||
- Leo revises the candidate rather than defending or silently applying it.
|
||||
- The resulting proposal packet is reviewable and replayable.
|
||||
- At least one negative control catches collusion, leaked row IDs, or unsupported agreement.
|
||||
|
||||
## Not done
|
||||
One model generating both sides in one prompt, a scripted transcript, or two agents sharing unrestricted memory is not agent-to-agent proof.
|
||||
|
||||
## Required tier
|
||||
T2 isolated multi-agent runtime proof; T3 handler execution is the next tier after local closure.
|
||||
|
||||
## Current truth
|
||||
The human-style three-turn chain works in one handler session. A live separated agent-as-user version has not been proven.
|
||||
|
||||
## Owned surface
|
||||
New challenger protocol runner, isolated session fixtures, verifier, and tests. Reuse benchmark contracts without editing the OOS scorer or Telegram sender.
|
||||
|
||||
## Forbidden
|
||||
No canonical apply, no hidden transcript injection, no shared writable memory, no public send, and no claim that simulated agents equal human acceptance.
|
||||
|
||||
## Required skills
|
||||
- `/Users/user/.codex/skills/subagent-wave-orchestration/SKILL.md`
|
||||
- `/Users/user/.codex/skills/capability-tier-proof/SKILL.md`
|
||||
- `/Users/user/.codex/skills/codex-benchmark-lab/SKILL.md`
|
||||
|
||||
## First action
|
||||
Create the platform Goal, define the session isolation contract and one minimal challenger/Leo canary, then run it with full traces before adding complexity.
|
||||
|
||||
## Delivery and proof
|
||||
Deliver a PR with the protocol, negative controls, retained transcript/trace receipt, tests, and explicit limitations.
|
||||
|
||||
## Platform Goal objective
|
||||
Complete and verify the isolated agent-as-user challenge and revision loop defined in this goal file.
|
||||
|
||||
## Worktree lifecycle
|
||||
At completion, report the delivery disposition and exact dirty state. Follow `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/goals/leo-fleet-20260715/WORKTREE_LIFECYCLE.md`. Keep the worktree while any useful commit, PR, patch, proof, or dirty delta remains unintegrated. The integration owner removes and prunes it only after merge, deliberate port, or explicit rejection is verified.
|
||||
|
|
@ -669,6 +669,10 @@ def verify_report(
|
|||
and roles.get("distinct_profile_roots") is True
|
||||
and challenger_role.get("process_identity_sha256")
|
||||
!= leo_role.get("process_identity_sha256")
|
||||
and challenger_role.get("profile_realpath_sha256")
|
||||
and leo_role.get("profile_realpath_sha256")
|
||||
and challenger_role.get("profile_realpath_sha256")
|
||||
!= leo_role.get("profile_realpath_sha256")
|
||||
and challenger_role.get("contract_sha256")
|
||||
== compute_runtime_role_contract_sha256(challenger_role)
|
||||
and leo_role.get("contract_sha256") == compute_runtime_role_contract_sha256(leo_role)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
|
|
@ -29,6 +30,25 @@ LEDGER_JSONL = REPORT_DIR / "leo-agent-challenger-loop-ledger.jsonl"
|
|||
DEFAULT_MODEL = "google/gemini-2.5-flash"
|
||||
DEFAULT_MAX_TOKENS = 768
|
||||
DEFAULT_REPORT_PREFIX = "leo-agent-challenger-loop"
|
||||
RETENTION_PROJECTION_SCHEMA = "livingip.leoAgentChallengerRetentionProjection.v1"
|
||||
BEHAVIOR_RETENTION_SCHEMA = "livingip.leoBehaviorManifestRetentionProjection.v1"
|
||||
RETENTION_FORBIDDEN_PATTERNS = (
|
||||
(
|
||||
"absolute runtime path",
|
||||
re.compile(r"(?<![A-Za-z0-9])/(?:Users|private|home|root|opt|tmp|var|etc)/[^\s\"']+"),
|
||||
),
|
||||
("PEM private key", re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")),
|
||||
("credentialed URL", re.compile(r"https?://[^/\s:@]+:[^/\s@]+@")),
|
||||
("JWT", re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b")),
|
||||
(
|
||||
"known token form",
|
||||
re.compile(
|
||||
r"\b(?:sk-[A-Za-z0-9_-]{20,}|AIza[0-9A-Za-z_-]{35}|gh[pousr]_[A-Za-z0-9]{20,}|"
|
||||
r"xox[baprs]-[A-Za-z0-9-]{20,})\b"
|
||||
),
|
||||
),
|
||||
("Bearer credential", re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]{20,}\b", re.IGNORECASE)),
|
||||
)
|
||||
|
||||
LEO_CHALLENGER_SOUL_OVERLAY = """
|
||||
|
||||
|
|
@ -1277,6 +1297,155 @@ def run_remote(
|
|||
return result
|
||||
|
||||
|
||||
def _sha256_text(value: Any) -> str:
|
||||
return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _manifest_content_projection(content: Any) -> dict[str, Any]:
|
||||
if not isinstance(content, dict):
|
||||
return {}
|
||||
return {
|
||||
"file_count": content.get("file_count"),
|
||||
"total_bytes": content.get("total_bytes"),
|
||||
"sha256": content.get("sha256"),
|
||||
"missing_count": len(content.get("missing") or []),
|
||||
"symlink_count": len(content.get("symlinks") or []),
|
||||
}
|
||||
|
||||
|
||||
def compact_behavior_manifest_for_retention(manifest: Any) -> dict[str, Any]:
|
||||
"""Keep behavior equality proof without retaining private per-file inventory."""
|
||||
|
||||
if not isinstance(manifest, dict):
|
||||
return {}
|
||||
projection = manifest.get("retention_projection")
|
||||
if isinstance(projection, dict) and projection.get("schema") == BEHAVIOR_RETENTION_SCHEMA:
|
||||
return copy.deepcopy(manifest)
|
||||
components: dict[str, Any] = {}
|
||||
removed_file_entries = 0
|
||||
removed_missing_entries = 0
|
||||
removed_symlink_entries = 0
|
||||
raw_components = manifest.get("components") if isinstance(manifest.get("components"), dict) else {}
|
||||
for name, raw_component in sorted(raw_components.items()):
|
||||
component = raw_component if isinstance(raw_component, dict) else {}
|
||||
content = component.get("content") if isinstance(component.get("content"), dict) else {}
|
||||
removed_file_entries += len(content.get("files") or [])
|
||||
removed_missing_entries += len(content.get("missing") or [])
|
||||
removed_symlink_entries += len(content.get("symlinks") or [])
|
||||
components[str(name)] = {
|
||||
key: copy.deepcopy(component.get(key))
|
||||
for key in ("role", "mutability", "replayability")
|
||||
}
|
||||
components[str(name)]["content"] = _manifest_content_projection(content)
|
||||
|
||||
runtimes: dict[str, Any] = {}
|
||||
for name in ("hermes_runtime", "teleo_infrastructure_runtime"):
|
||||
runtime = manifest.get(name) if isinstance(manifest.get(name), dict) else {}
|
||||
source_tree = runtime.get("source_tree") if isinstance(runtime.get("source_tree"), dict) else {}
|
||||
removed_file_entries += len(source_tree.get("files") or [])
|
||||
removed_missing_entries += len(source_tree.get("missing") or [])
|
||||
runtimes[name] = {
|
||||
"git_head": runtime.get("git_head"),
|
||||
"git_state": copy.deepcopy(runtime.get("git_state")),
|
||||
"source_tree": _manifest_content_projection(source_tree),
|
||||
}
|
||||
|
||||
removed_root_count = sum(
|
||||
1 for name in ("profile_root", "hermes_root", "deployment_root") if manifest.get(name)
|
||||
)
|
||||
return {
|
||||
"schema": manifest.get("schema"),
|
||||
"generated_at_utc": manifest.get("generated_at_utc"),
|
||||
"behavior_sha256": manifest.get("behavior_sha256"),
|
||||
"model_runtime": copy.deepcopy(manifest.get("model_runtime")),
|
||||
"canonical_database": copy.deepcopy(manifest.get("canonical_database")),
|
||||
"components": components,
|
||||
**runtimes,
|
||||
"retention_projection": {
|
||||
"schema": BEHAVIOR_RETENTION_SCHEMA,
|
||||
"raw_manifest_sha256": protocol.canonical_sha256(manifest),
|
||||
"absolute_root_values_removed": True,
|
||||
"absolute_root_values_removed_count": removed_root_count,
|
||||
"per_file_inventories_removed": True,
|
||||
"removed_file_entries": removed_file_entries,
|
||||
"removed_missing_entries": removed_missing_entries,
|
||||
"removed_symlink_entries": removed_symlink_entries,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def sanitize_report_for_retention(report: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Project a live report into a review-safe, independently verifiable receipt."""
|
||||
|
||||
retained = copy.deepcopy(report)
|
||||
projection = retained.get("retention_projection")
|
||||
if isinstance(projection, dict) and projection.get("schema") == RETENTION_PROJECTION_SCHEMA:
|
||||
assert_report_safe_for_retention(retained)
|
||||
return retained
|
||||
pre_projection_report_sha256 = protocol.canonical_sha256(retained)
|
||||
behavior_projection: dict[str, Any] = {}
|
||||
for field in ("live_behavior_manifest_before", "live_behavior_manifest_after"):
|
||||
raw_manifest = retained.get(field)
|
||||
compacted = compact_behavior_manifest_for_retention(raw_manifest)
|
||||
retained[field] = compacted
|
||||
behavior_projection[field] = copy.deepcopy(compacted.get("retention_projection"))
|
||||
|
||||
redacted_fields: list[str] = []
|
||||
remote_report_path = retained.pop("remote_report_path", None)
|
||||
if remote_report_path:
|
||||
redacted_fields.append("remote_report_path")
|
||||
|
||||
transport = retained.get("execution_transport")
|
||||
if isinstance(transport, dict) and isinstance(transport.get("recovery"), dict):
|
||||
recovery = transport["recovery"]
|
||||
remote_path = recovery.pop("remote_path", None)
|
||||
if remote_path:
|
||||
redacted_fields.append("execution_transport.recovery.remote_path")
|
||||
|
||||
original_role_contracts: dict[str, str] = {}
|
||||
profile_realpath_hash_verified: dict[str, bool] = {}
|
||||
isolation = retained.get("isolation") if isinstance(retained.get("isolation"), dict) else {}
|
||||
roles = isolation.get("roles") if isinstance(isolation.get("roles"), dict) else {}
|
||||
for name, role in sorted(roles.items()):
|
||||
if not isinstance(role, dict):
|
||||
continue
|
||||
original_role_contracts[str(name)] = str(role.get("contract_sha256") or "")
|
||||
profile_realpath = role.get("profile_realpath")
|
||||
if profile_realpath:
|
||||
matches = _sha256_text(profile_realpath) == role.get("profile_realpath_sha256")
|
||||
profile_realpath_hash_verified[str(name)] = matches
|
||||
if not matches:
|
||||
raise ValueError(f"{name} profile realpath hash does not match before retention")
|
||||
redacted_fields.append(f"isolation.roles.{name}.profile_realpath")
|
||||
role["profile_realpath"] = None
|
||||
role["contract_sha256"] = protocol.compute_runtime_role_contract_sha256(role)
|
||||
|
||||
recovery = transport.get("recovery") if isinstance(transport, dict) else {}
|
||||
retained["retention_projection"] = {
|
||||
"schema": RETENTION_PROJECTION_SCHEMA,
|
||||
"raw_remote_report_sha256": (
|
||||
recovery.get("recovered_report_sha256") if isinstance(recovery, dict) else None
|
||||
),
|
||||
"pre_projection_report_sha256": pre_projection_report_sha256,
|
||||
"behavior_manifests": behavior_projection,
|
||||
"redacted_fields": sorted(redacted_fields),
|
||||
"redacted_field_count": len(redacted_fields),
|
||||
"profile_realpath_hash_verified": profile_realpath_hash_verified,
|
||||
"original_runtime_role_contract_sha256": original_role_contracts,
|
||||
"absolute_runtime_paths_removed": True,
|
||||
"verifier_contract_recomputed_after_redaction": True,
|
||||
}
|
||||
assert_report_safe_for_retention(retained)
|
||||
return retained
|
||||
|
||||
|
||||
def assert_report_safe_for_retention(report: dict[str, Any]) -> None:
|
||||
encoded = json.dumps(report, sort_keys=True, ensure_ascii=True)
|
||||
for label, pattern in RETENTION_FORBIDDEN_PATTERNS:
|
||||
if pattern.search(encoded):
|
||||
raise ValueError(f"retained report contains forbidden {label}")
|
||||
|
||||
|
||||
def write_outputs(remote: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
report = remote.get("parsed") if isinstance(remote.get("parsed"), dict) else {}
|
||||
|
|
@ -1285,6 +1454,7 @@ def write_outputs(remote: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any
|
|||
report["execution_transport"] = remote.get("execution_transport")
|
||||
if remote.get("stderr"):
|
||||
report["remote_stderr"] = remote["stderr"]
|
||||
report = sanitize_report_for_retention(report)
|
||||
OUTPUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
source_sha = hashlib.sha256(OUTPUT_JSON.read_bytes()).hexdigest()
|
||||
receipt = protocol.verify_report(report, source_report_sha256=source_sha)
|
||||
|
|
|
|||
|
|
@ -373,6 +373,18 @@ def test_same_handler_or_model_session_fails_separation() -> None:
|
|||
assert receipt["checks"]["distinct_model_session_identities"] is False
|
||||
|
||||
|
||||
def test_same_profile_root_hash_fails_separation_without_requiring_raw_paths() -> None:
|
||||
report = passing_report()
|
||||
challenger = report["isolation"]["roles"]["challenger"]
|
||||
leo = report["isolation"]["roles"]["leo"]
|
||||
leo["profile_realpath_sha256"] = challenger["profile_realpath_sha256"]
|
||||
leo["contract_sha256"] = protocol.compute_runtime_role_contract_sha256(leo)
|
||||
|
||||
receipt = protocol.verify_report(report, source_report_sha256="4" * 64)
|
||||
|
||||
assert receipt["checks"]["separate_process_and_profile_roots"] is False
|
||||
|
||||
|
||||
def test_repeated_execution_hashes_fail_even_with_valid_parent_transcript() -> None:
|
||||
report = passing_report()
|
||||
for turn in report["turns"]:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts import run_leo_agent_challenger_loop as runner
|
||||
|
|
@ -30,3 +32,130 @@ def test_remote_script_rejects_unbounded_tokens_and_unsafe_report_prefix() -> No
|
|||
|
||||
with pytest.raises(ValueError, match="report_prefix"):
|
||||
runner.build_remote_script("abc123", report_prefix="bad/path")
|
||||
|
||||
|
||||
def test_retention_projection_removes_runtime_paths_without_losing_hash_proof() -> None:
|
||||
manifest = {
|
||||
"schema": "livingip.leoBehaviorManifest.v1",
|
||||
"generated_at_utc": "2026-07-15T00:00:00+00:00",
|
||||
"behavior_sha256": "a" * 64,
|
||||
"profile_root": "/home/teleo/.hermes/profiles/leoclean",
|
||||
"hermes_root": "/home/teleo/.hermes/hermes-agent",
|
||||
"deployment_root": "/opt/teleo-eval/workspaces/deploy-infra",
|
||||
"model_runtime": {"status": "observed"},
|
||||
"canonical_database": {"included": False},
|
||||
"components": {
|
||||
"conversation_sessions": {
|
||||
"role": "session state",
|
||||
"mutability": "dynamic",
|
||||
"replayability": "hashable",
|
||||
"content": {
|
||||
"files": [{"path": "/home/teleo/private/session.json", "sha256": "b" * 64}],
|
||||
"missing": [],
|
||||
"symlinks": [],
|
||||
"file_count": 1,
|
||||
"total_bytes": 42,
|
||||
"sha256": "c" * 64,
|
||||
},
|
||||
}
|
||||
},
|
||||
"hermes_runtime": {
|
||||
"git_head": "d" * 40,
|
||||
"git_state": {"worktree_clean": True},
|
||||
"source_tree": {
|
||||
"files": [{"path": "/home/teleo/private/runtime.py", "sha256": "e" * 64}],
|
||||
"missing": [],
|
||||
"file_count": 1,
|
||||
"total_bytes": 84,
|
||||
"sha256": "f" * 64,
|
||||
},
|
||||
},
|
||||
"teleo_infrastructure_runtime": {
|
||||
"git_head": "1" * 40,
|
||||
"git_state": {"worktree_clean": True},
|
||||
"source_tree": {
|
||||
"files": [{"path": "/opt/teleo-eval/private/runner.py", "sha256": "2" * 64}],
|
||||
"missing": [],
|
||||
"file_count": 1,
|
||||
"total_bytes": 126,
|
||||
"sha256": "3" * 64,
|
||||
},
|
||||
},
|
||||
}
|
||||
role = {
|
||||
"role": "leo",
|
||||
"runtime_kind": "gatewayrunner_temp_profile_no_model_tools",
|
||||
"profile_realpath": "/tmp/leo-agent-challenger-loop-test/leo-profile",
|
||||
}
|
||||
role["profile_realpath_sha256"] = runner._sha256_text(role["profile_realpath"])
|
||||
role["contract_sha256"] = runner.protocol.compute_runtime_role_contract_sha256(role)
|
||||
original_contract = role["contract_sha256"]
|
||||
raw = {
|
||||
"remote_report_path": "/tmp/leo-agent-challenger-loop-test.json",
|
||||
"execution_transport": {
|
||||
"recovery": {
|
||||
"remote_path": "/tmp/leo-agent-challenger-loop-test.json",
|
||||
"recovered_report_sha256": "5" * 64,
|
||||
}
|
||||
},
|
||||
"isolation": {"roles": {"leo": role}},
|
||||
"live_behavior_manifest_before": manifest,
|
||||
"live_behavior_manifest_after": manifest,
|
||||
}
|
||||
|
||||
retained = runner.sanitize_report_for_retention(raw)
|
||||
encoded = json.dumps(retained, sort_keys=True)
|
||||
|
||||
assert "/home/" not in encoded
|
||||
assert "/opt/" not in encoded
|
||||
assert "/tmp/" not in encoded
|
||||
assert raw["remote_report_path"].startswith("/tmp/")
|
||||
assert retained["live_behavior_manifest_before"]["components"]["conversation_sessions"][
|
||||
"content"
|
||||
] == {
|
||||
"file_count": 1,
|
||||
"missing_count": 0,
|
||||
"sha256": "c" * 64,
|
||||
"symlink_count": 0,
|
||||
"total_bytes": 42,
|
||||
}
|
||||
assert retained["live_behavior_manifest_before"]["retention_projection"][
|
||||
"removed_file_entries"
|
||||
] == 3
|
||||
retained_role = retained["isolation"]["roles"]["leo"]
|
||||
assert retained_role["profile_realpath"] is None
|
||||
assert retained_role["profile_realpath_sha256"] == runner._sha256_text(
|
||||
"/tmp/leo-agent-challenger-loop-test/leo-profile"
|
||||
)
|
||||
assert retained_role["contract_sha256"] == runner.protocol.compute_runtime_role_contract_sha256(
|
||||
retained_role
|
||||
)
|
||||
assert retained["retention_projection"]["original_runtime_role_contract_sha256"]["leo"] == (
|
||||
original_contract
|
||||
)
|
||||
assert retained["retention_projection"]["raw_remote_report_sha256"] == "5" * 64
|
||||
assert retained["retention_projection"]["profile_realpath_hash_verified"] == {"leo": True}
|
||||
assert runner.sanitize_report_for_retention(retained) == retained
|
||||
retained["remote_stderr"] = "traceback from /home/teleo/private/runtime.py"
|
||||
with pytest.raises(ValueError, match="forbidden absolute runtime path"):
|
||||
runner.sanitize_report_for_retention(retained)
|
||||
|
||||
|
||||
def test_retention_projection_fails_closed_on_mismatched_path_hash_or_unexpected_path() -> None:
|
||||
mismatched_role = {
|
||||
"role": "leo",
|
||||
"profile_realpath": "/tmp/leo-profile",
|
||||
"profile_realpath_sha256": "0" * 64,
|
||||
}
|
||||
mismatched_role["contract_sha256"] = runner.protocol.compute_runtime_role_contract_sha256(
|
||||
mismatched_role
|
||||
)
|
||||
with pytest.raises(ValueError, match="profile realpath hash does not match"):
|
||||
runner.sanitize_report_for_retention(
|
||||
{"isolation": {"roles": {"leo": mismatched_role}}}
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="forbidden absolute runtime path"):
|
||||
runner.sanitize_report_for_retention(
|
||||
{"remote_stderr": "traceback from /home/teleo/private/runtime.py"}
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue