161 lines
6.5 KiB
Python
161 lines
6.5 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from scripts import run_leo_agent_challenger_loop as runner
|
|
|
|
|
|
def test_remote_script_embeds_stateless_challenger_read_only_guard_and_bounded_context() -> None:
|
|
script = runner.build_remote_script("abc123", challenger_max_tokens=512)
|
|
|
|
assert "stateless_openrouter_agent" in script
|
|
assert "gatewayrunner_temp_profile_no_model_tools" in script
|
|
assert "blocked by isolated challenger-loop read-only guard" in script
|
|
context_source = runner._patched_db_context_source()
|
|
assert '"--limit",\n "4",' in context_source
|
|
assert '"--context-limit",\n "6",' in context_source
|
|
assert "CHALLENGER_MAX_TOKENS = 512" in script
|
|
assert "posted_to_telegram" in script
|
|
assert "db_fingerprint_unchanged" in script
|
|
assert '"temperature": 0.0' in script
|
|
assert "isolated_challenger_final_revision" in script
|
|
assert '"use_database_context": step < 3' in script
|
|
assert "LEO_CHALLENGER_CONTEXT_ONLY" in script
|
|
assert "Follow the challenger's requested labels exactly" in script
|
|
|
|
|
|
def test_remote_script_rejects_unbounded_tokens_and_unsafe_report_prefix() -> None:
|
|
with pytest.raises(ValueError, match="between 128 and 2048"):
|
|
runner.build_remote_script("abc123", challenger_max_tokens=4096)
|
|
|
|
with pytest.raises(ValueError, match="report_prefix"):
|
|
runner.build_remote_script("abc123", report_prefix="bad/path")
|
|
|
|
|
|
def test_retention_projection_removes_runtime_paths_without_losing_hash_proof() -> None:
|
|
manifest = {
|
|
"schema": "livingip.leoBehaviorManifest.v1",
|
|
"generated_at_utc": "2026-07-15T00:00:00+00:00",
|
|
"behavior_sha256": "a" * 64,
|
|
"profile_root": "/home/teleo/.hermes/profiles/leoclean",
|
|
"hermes_root": "/home/teleo/.hermes/hermes-agent",
|
|
"deployment_root": "/opt/teleo-eval/workspaces/deploy-infra",
|
|
"model_runtime": {"status": "observed"},
|
|
"canonical_database": {"included": False},
|
|
"components": {
|
|
"conversation_sessions": {
|
|
"role": "session state",
|
|
"mutability": "dynamic",
|
|
"replayability": "hashable",
|
|
"content": {
|
|
"files": [{"path": "/home/teleo/private/session.json", "sha256": "b" * 64}],
|
|
"missing": [],
|
|
"symlinks": [],
|
|
"file_count": 1,
|
|
"total_bytes": 42,
|
|
"sha256": "c" * 64,
|
|
},
|
|
}
|
|
},
|
|
"hermes_runtime": {
|
|
"git_head": "d" * 40,
|
|
"git_state": {"worktree_clean": True},
|
|
"source_tree": {
|
|
"files": [{"path": "/home/teleo/private/runtime.py", "sha256": "e" * 64}],
|
|
"missing": [],
|
|
"file_count": 1,
|
|
"total_bytes": 84,
|
|
"sha256": "f" * 64,
|
|
},
|
|
},
|
|
"teleo_infrastructure_runtime": {
|
|
"git_head": "1" * 40,
|
|
"git_state": {"worktree_clean": True},
|
|
"source_tree": {
|
|
"files": [{"path": "/opt/teleo-eval/private/runner.py", "sha256": "2" * 64}],
|
|
"missing": [],
|
|
"file_count": 1,
|
|
"total_bytes": 126,
|
|
"sha256": "3" * 64,
|
|
},
|
|
},
|
|
}
|
|
role = {
|
|
"role": "leo",
|
|
"runtime_kind": "gatewayrunner_temp_profile_no_model_tools",
|
|
"profile_realpath": "/tmp/leo-agent-challenger-loop-test/leo-profile",
|
|
}
|
|
role["profile_realpath_sha256"] = runner._sha256_text(role["profile_realpath"])
|
|
role["contract_sha256"] = runner.protocol.compute_runtime_role_contract_sha256(role)
|
|
original_contract = role["contract_sha256"]
|
|
raw = {
|
|
"remote_report_path": "/tmp/leo-agent-challenger-loop-test.json",
|
|
"execution_transport": {
|
|
"recovery": {
|
|
"remote_path": "/tmp/leo-agent-challenger-loop-test.json",
|
|
"recovered_report_sha256": "5" * 64,
|
|
}
|
|
},
|
|
"isolation": {"roles": {"leo": role}},
|
|
"live_behavior_manifest_before": manifest,
|
|
"live_behavior_manifest_after": manifest,
|
|
}
|
|
|
|
retained = runner.sanitize_report_for_retention(raw)
|
|
encoded = json.dumps(retained, sort_keys=True)
|
|
|
|
assert "/home/" not in encoded
|
|
assert "/opt/" not in encoded
|
|
assert "/tmp/" not in encoded
|
|
assert raw["remote_report_path"].startswith("/tmp/")
|
|
assert retained["live_behavior_manifest_before"]["components"]["conversation_sessions"][
|
|
"content"
|
|
] == {
|
|
"file_count": 1,
|
|
"missing_count": 0,
|
|
"sha256": "c" * 64,
|
|
"symlink_count": 0,
|
|
"total_bytes": 42,
|
|
}
|
|
assert retained["live_behavior_manifest_before"]["retention_projection"][
|
|
"removed_file_entries"
|
|
] == 3
|
|
retained_role = retained["isolation"]["roles"]["leo"]
|
|
assert retained_role["profile_realpath"] is None
|
|
assert retained_role["profile_realpath_sha256"] == runner._sha256_text(
|
|
"/tmp/leo-agent-challenger-loop-test/leo-profile"
|
|
)
|
|
assert retained_role["contract_sha256"] == runner.protocol.compute_runtime_role_contract_sha256(
|
|
retained_role
|
|
)
|
|
assert retained["retention_projection"]["original_runtime_role_contract_sha256"]["leo"] == (
|
|
original_contract
|
|
)
|
|
assert retained["retention_projection"]["raw_remote_report_sha256"] == "5" * 64
|
|
assert retained["retention_projection"]["profile_realpath_hash_verified"] == {"leo": True}
|
|
assert runner.sanitize_report_for_retention(retained) == retained
|
|
retained["remote_stderr"] = "traceback from /home/teleo/private/runtime.py"
|
|
with pytest.raises(ValueError, match="forbidden absolute runtime path"):
|
|
runner.sanitize_report_for_retention(retained)
|
|
|
|
|
|
def test_retention_projection_fails_closed_on_mismatched_path_hash_or_unexpected_path() -> None:
|
|
mismatched_role = {
|
|
"role": "leo",
|
|
"profile_realpath": "/tmp/leo-profile",
|
|
"profile_realpath_sha256": "0" * 64,
|
|
}
|
|
mismatched_role["contract_sha256"] = runner.protocol.compute_runtime_role_contract_sha256(
|
|
mismatched_role
|
|
)
|
|
with pytest.raises(ValueError, match="profile realpath hash does not match"):
|
|
runner.sanitize_report_for_retention(
|
|
{"isolation": {"roles": {"leo": mismatched_role}}}
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="forbidden absolute runtime path"):
|
|
runner.sanitize_report_for_retention(
|
|
{"remote_stderr": "traceback from /home/teleo/private/runtime.py"}
|
|
)
|