teleo/teleo-infrastructure
8
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

Harden OOS benchmark provenance and leakage checks

Browse source
This commit is contained in:
twentyOne2x 2026-07-15 12:06:35 +02:00
parent 93eddbcd4f
commit 8a4325f42a
16 changed files with 319 additions and 1625 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1277
docs/reports/leo-oos-reasoning-benchmark-20260715/blinded-protocol.json
View file

File diff suppressed because it is too large Load diff

17
docs/reports/leo-oos-reasoning-benchmark-20260715/evaluated-protocol-manifest.json Normal file
View file

@ -0,0 +1,17 @@
{
"schema": "livingip.leoEvaluatedOosProtocolManifest.v1",
"protocol_schema": "livingip.leoM3taversalOosProtocol.v2",
"protocol_id": "leo-m3taversal-oos-8c0fc263b5cd95b8",
"protocol_hash_sha256": "d68330b20d31677b231beb8450d5fdfd40efebf498cfc2f8ae2e63efd860bf43",
"protocol_file_sha256": "fac462ee83b1035215cca12262e27b70c70e8a8ed0a216aa70c69ef46cd6a185",
"harness_git_head": "93eddbcd4f27bd9e349f217767da2c9bf2b7baeb",
"trial_count": 3,
"prompt_count": 30,
"family_count": 10,
"unique_prompt_id_count": 30,
"unique_message_hash_count": 30,
"validation_pass": true,
"prompt_bodies_committed": false,
"retention_contract": "The exact post-evaluation protocol is retained as a private mode-0600 artifact and is bound by protocol_file_sha256. This manifest is not sufficient to recreate prompt bodies by itself.",
"claim_ceiling": "This manifest identifies the exact evaluated protocol without publishing the formerly blinded prompts. It does not independently authenticate live VPS execution."
}

46
hermes-agent/leoclean-bin/kb_tool.py
View file

@ -203,6 +203,7 @@ STOPWORDS = {
"with", "with",
} }
def parse_args() -> argparse.Namespace: def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False)
parser.add_argument("--ssh", default="teleo@77.42.65.182") parser.add_argument("--ssh", default="teleo@77.42.65.182")
@ -461,8 +462,10 @@ def _has_unnegated_database_state_request(query: str) -> bool:
segment, segment,
) )
scope_start = scope_match.start() if scope_match else -1 scope_start = scope_match.start() if scope_match else -1
if scope_start < 0 and "canonical" in segment and any( if (
term in segment for term in ("approved", "applied", "proposal") scope_start < 0
and "canonical" in segment
and any(term in segment for term in ("approved", "applied", "proposal"))
): ):
scope_start = segment.index("canonical") scope_start = segment.index("canonical")
negation_before_scope = any( negation_before_scope = any(
@ -522,11 +525,7 @@ def rank_query_matching_proposals(
ranked: list[tuple[int, int, str, int, dict[str, Any], list[str]]] = [] ranked: list[tuple[int, int, str, int, dict[str, Any], list[str]]] = []
for index, proposal in enumerate(proposals): for index, proposal in enumerate(proposals):
haystack = json.dumps(proposal, sort_keys=True, default=str).lower() haystack = json.dumps(proposal, sort_keys=True, default=str).lower()
matched_terms = [ matched_terms = [term for term in terms if re.search(rf"(?<![a-z0-9]){re.escape(term)}(?![a-z0-9])", haystack)]
term
for term in terms
if re.search(rf"(?<![a-z0-9]){re.escape(term)}(?![a-z0-9])", haystack)
]
if len(matched_terms) < minimum_term_matches or not version_terms.issubset(matched_terms): if len(matched_terms) < minimum_term_matches or not version_terms.issubset(matched_terms):
continue continue
score = sum(2 if len(term) >= 8 or re.fullmatch(r"v\d+", term) else 1 for term in matched_terms) score = sum(2 if len(term) >= 8 or re.fullmatch(r"v\d+", term) else 1 for term in matched_terms)
@ -591,8 +590,7 @@ def operational_contracts(
) )
) )
and any( and any(
term in lowered term in lowered for term in ("wrong", "missing", "stuck", "link", "point", "canonical evidence", "audit")
for term in ("wrong", "missing", "stuck", "link", "point", "canonical evidence", "audit")
) )
) )
kb_scope = ( kb_scope = (
@ -635,9 +633,7 @@ def operational_contracts(
) )
) )
) )
broad_kb_change_question = _has_unnegated_action( broad_kb_change_question = _has_unnegated_action(query, ("change", "changed", "update", "updated", "landed"))
query, ("change", "changed", "update", "updated", "landed")
)
proposal_state_question = ( proposal_state_question = (
kb_scope kb_scope
and not forecast_resolution_question and not forecast_resolution_question
@ -936,9 +932,7 @@ def operational_contracts(
contracts.append( contracts.append(
{ {
"id": "forecast_resolution_schema", "id": "forecast_resolution_schema",
"current_columns": { "current_columns": {name: list(schema.get(name, ())) for name in ("claims", "claim_edges")},
name: list(schema.get(name, ())) for name in ("claims", "claim_edges")
},
"history_boundary": "preserve the original forecast and its missing resolution criteria", "history_boundary": "preserve the original forecast and its missing resolution criteria",
"schema_boundary": ( "schema_boundary": (
"do not invent forecast-resolution fields or a resolves edge; stage a reviewed schema proposal" "do not invent forecast-resolution fields or a resolves edge; stage a reviewed schema proposal"
@ -955,9 +949,7 @@ def operational_contracts(
contracts.append( contracts.append(
{ {
"id": "claim_supersession_schema", "id": "claim_supersession_schema",
"current_columns": { "current_columns": {name: list(schema.get(name, ())) for name in ("claims", "claim_edges")},
name: list(schema.get(name, ())) for name in ("claims", "claim_edges")
},
"insert_boundary": "approve_claim may insert a replacement claim plus a claim-to-claim edge", "insert_boundary": "approve_claim may insert a replacement claim plus a claim-to-claim edge",
"update_boundary": ( "update_boundary": (
"existing claim status and superseded_by updates require a separate reviewed apply capability" "existing claim status and superseded_by updates require a separate reviewed apply capability"
@ -967,7 +959,9 @@ def operational_contracts(
telegram_delivery_question = ( telegram_delivery_question = (
"telegram" in lowered "telegram" in lowered
and any(term in lowered for term in ("gatewayrunner", "temporary-profile", "temporary profile", "posted nothing")) and any(
term in lowered for term in ("gatewayrunner", "temporary-profile", "temporary profile", "posted nothing")
)
and any(term in lowered for term in ("proven", "delivery", "visible")) and any(term in lowered for term in ("proven", "delivery", "visible"))
) )
if telegram_delivery_question: if telegram_delivery_question:
@ -1307,7 +1301,9 @@ def compile_operational_response(contracts: list[dict[str, Any]]) -> str | None:
) )
elif status == "approved": elif status == "approved":
lead = "No." lead = "No."
state_sentence = "The matching proposal is reviewer-approved, but applied_at is empty; approved is not applied." state_sentence = (
"The matching proposal is reviewer-approved, but applied_at is empty; approved is not applied."
)
elif status == "pending_review": elif status == "pending_review":
lead = "No." lead = "No."
state_sentence = ( state_sentence = (
@ -2427,11 +2423,13 @@ def prepare_source(args: argparse.Namespace) -> dict[str, Any]:
def propose_source(args: argparse.Namespace) -> dict[str, Any]: def propose_source(args: argparse.Namespace) -> dict[str, Any]:
if not args.local: if not args.local:
raise SystemExit( raise SystemExit("propose-source is VPS-local only; run it through the deployed teleo-kb wrapper on the VPS")
"propose-source is VPS-local only; run it through the deployed teleo-kb wrapper on the VPS"
)
receipt_path = args.receipt.expanduser().resolve() receipt_path = args.receipt.expanduser().resolve()
input_paths = {args.artifact.expanduser().resolve(), args.text.expanduser().resolve(), args.manifest.expanduser().resolve()} input_paths = {
args.artifact.expanduser().resolve(),
args.text.expanduser().resolve(),
args.manifest.expanduser().resolve(),
}
if receipt_path in input_paths: if receipt_path in input_paths:
raise SystemExit("--receipt must not overwrite an input file") raise SystemExit("--receipt must not overwrite an input file")

6
hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py
View file

@ -858,11 +858,7 @@ def _post_llm_call(**kwargs: Any) -> dict[str, str] | None:
else: else:
delivered = response delivered = response
hard_max = _reply_hard_max(contracts) hard_max = _reply_hard_max(contracts)
budget_compacted = bool( budget_compacted = bool(not fail_closed and hard_max is not None and _word_count(delivered) > hard_max)
not fail_closed
and hard_max is not None
and _word_count(delivered) > hard_max
)
if budget_compacted: if budget_compacted:
delivered = _compact_response_to_budget(delivered, hard_max) delivered = _compact_response_to_budget(delivered, hard_max)
delivered_issues = response_contract_issues(delivered, contracts) delivered_issues = response_contract_issues(delivered, contracts)

16
scripts/leo_tool_trace.py
View file

@ -33,9 +33,7 @@ MUTATING_SUBCOMMANDS = frozenset(
) )
KNOWN_SUBCOMMANDS = READ_ONLY_SUBCOMMANDS | MUTATING_SUBCOMMANDS KNOWN_SUBCOMMANDS = READ_ONLY_SUBCOMMANDS | MUTATING_SUBCOMMANDS
UUID_RE = re.compile( UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b")
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b"
)
SHA256_RE = re.compile(r"\b[0-9a-fA-F]{64}\b") SHA256_RE = re.compile(r"\b[0-9a-fA-F]{64}\b")
TELEO_KB_COMMAND_RE = re.compile( TELEO_KB_COMMAND_RE = re.compile(
r"(?:^|[\s;&|()])(?:[^\s;&|()]+/)?teleo-kb\s+(?P<subcommand>[a-z][a-z0-9-]*)\b", r"(?:^|[\s;&|()])(?:[^\s;&|()]+/)?teleo-kb\s+(?P<subcommand>[a-z][a-z0-9-]*)\b",
@ -217,11 +215,7 @@ def extract_kb_tool_trace(messages: list[dict[str, Any]] | None) -> dict[str, An
record["result"] = _sanitize_result(message.get("content")) record["result"] = _sanitize_result(message.get("content"))
completed = [ completed = [
call call for call in calls if call["result"] and call["result"]["nonempty"] and not call["result"]["error_detected"]
for call in calls
if call["result"]
and call["result"]["nonempty"]
and not call["result"]["error_detected"]
] ]
receipt_calls = [ receipt_calls = [
call call
@ -230,11 +224,7 @@ def extract_kb_tool_trace(messages: list[dict[str, Any]] | None) -> dict[str, An
and call["result"]["retrieval_receipt"].get("semantic_context_sha256") and call["result"]["retrieval_receipt"].get("semantic_context_sha256")
and call["result"]["retrieval_receipt"].get("artifact_state_sha256") and call["result"]["retrieval_receipt"].get("artifact_state_sha256")
] ]
access_modes = { access_modes = {invocation["access_mode"] for call in calls for invocation in call["database_invocations"]}
invocation["access_mode"]
for call in calls
for invocation in call["database_invocations"]
}
return { return {
"schema": "livingip.leoKbToolTrace.v1", "schema": "livingip.leoKbToolTrace.v1",
"database_tool_call_count": len(calls), "database_tool_call_count": len(calls),

38
scripts/leo_turn_execution_manifest.py
View file

@ -112,9 +112,7 @@ def _safe_usage(value: Any) -> dict[str, int | float | None]:
def _trace_events(result: dict[str, Any], event: str) -> list[dict[str, Any]]: def _trace_events(result: dict[str, Any], event: str) -> list[dict[str, Any]]:
return [ return [
item item for item in result.get("model_call_trace") or [] if isinstance(item, dict) and item.get("event") == event
for item in result.get("model_call_trace") or []
if isinstance(item, dict) and item.get("event") == event
] ]
@ -224,9 +222,7 @@ def _context_receipts(result: dict[str, Any], *, expected_query_sha256: str) ->
or record.get("query_sha256") != expected_query_sha256 or record.get("query_sha256") != expected_query_sha256
): ):
continue continue
receipt = _safe_retrieval_receipt( receipt = _safe_retrieval_receipt(record.get("retrieval_receipt"), expected_query_sha256=expected_query_sha256)
record.get("retrieval_receipt"), expected_query_sha256=expected_query_sha256
)
if receipt is not None: if receipt is not None:
receipts.append(receipt) receipts.append(receipt)
return receipts return receipts
@ -238,7 +234,7 @@ def _tool_receipts(result: dict[str, Any]) -> list[dict[str, Any]]:
for call in trace.get("calls") or []: for call in trace.get("calls") or []:
if not isinstance(call, dict): if not isinstance(call, dict):
continue continue
raw = ((call.get("result") or {}).get("retrieval_receipt") or {}) raw = (call.get("result") or {}).get("retrieval_receipt") or {}
if ( if (
not isinstance(raw, dict) not isinstance(raw, dict)
or raw.get("schema") != RETRIEVAL_RECEIPT_SCHEMA or raw.get("schema") != RETRIEVAL_RECEIPT_SCHEMA
@ -273,9 +269,7 @@ def _database_required(result: dict[str, Any], *, prompt: str) -> bool:
return bool(trace.get("database_tool_call_count")) return bool(trace.get("database_tool_call_count"))
def _database_context_binding( def _database_context_binding(result: dict[str, Any], *, prompt_sha256: str, reply_sha256: str) -> dict[str, Any]:
result: dict[str, Any], *, prompt_sha256: str, reply_sha256: str
) -> dict[str, Any]:
records = [item for item in result.get("database_context_trace") or [] if isinstance(item, dict)] records = [item for item in result.get("database_context_trace") or [] if isinstance(item, dict)]
pre_records = [item for item in records if item.get("event") == "pre_llm_call"] pre_records = [item for item in records if item.get("event") == "pre_llm_call"]
post_records = [item for item in records if item.get("event") == "post_llm_call"] post_records = [item for item in records if item.get("event") == "post_llm_call"]
@ -371,8 +365,7 @@ def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]:
and all( and all(
call.get("database_invocations") call.get("database_invocations")
and all( and all(
invocation.get("access_mode") == "read_only" invocation.get("access_mode") == "read_only" and _is_sha256(invocation.get("command_sha256"))
and _is_sha256(invocation.get("command_sha256"))
for invocation in call.get("database_invocations") or [] for invocation in call.get("database_invocations") or []
) )
for call in calls for call in calls
@ -433,10 +426,7 @@ def _conversation_binding(
before = result.get("conversation_before") if isinstance(result.get("conversation_before"), dict) else {} before = result.get("conversation_before") if isinstance(result.get("conversation_before"), dict) else {}
after = result.get("conversation_after") if isinstance(result.get("conversation_after"), dict) else {} after = result.get("conversation_after") if isinstance(result.get("conversation_after"), dict) else {}
expected = expected_previous_conversation or {} expected = expected_previous_conversation or {}
before_valid = bool( before_valid = bool(_non_negative_int(before.get("message_count")) and _is_sha256(before.get("messages_sha256")))
_non_negative_int(before.get("message_count"))
and _is_sha256(before.get("messages_sha256"))
)
after_valid = bool( after_valid = bool(
_non_negative_int(after.get("message_count")) _non_negative_int(after.get("message_count"))
and after.get("message_count", 0) > before.get("message_count", -1) and after.get("message_count", 0) > before.get("message_count", -1)
@ -514,12 +504,9 @@ def _model_execution_binding(
and post.get("prompt_sha256") == prompt_sha256 and post.get("prompt_sha256") == prompt_sha256
), ),
"raw_response_bound": bool( "raw_response_bound": bool(
_is_sha256(raw_response_sha256) _is_sha256(raw_response_sha256) and post.get("raw_assistant_response_sha256") == raw_response_sha256
and post.get("raw_assistant_response_sha256") == raw_response_sha256
),
"delivered_response_bound": bool(
response_hashes_valid and responses[-1].get("content_sha256") == reply_sha256
), ),
"delivered_response_bound": bool(response_hashes_valid and responses[-1].get("content_sha256") == reply_sha256),
"response_trace_count_matches_api_calls": len(responses) == len(calls), "response_trace_count_matches_api_calls": len(responses) == len(calls),
"api_call_sequence_valid": call_sequence == list(range(1, len(calls) + 1)), "api_call_sequence_valid": call_sequence == list(range(1, len(calls) + 1)),
"session_binding_valid": len(session_hashes) == 1 and all(_is_sha256(item) for item in session_hashes), "session_binding_valid": len(session_hashes) == 1 and all(_is_sha256(item) for item in session_hashes),
@ -575,8 +562,7 @@ def _required_missing(
), ),
"harness_git_head": _is_git_sha(harness_source.get("git_head")), "harness_git_head": _is_git_sha(harness_source.get("git_head")),
"harness_worktree_clean": bool( "harness_worktree_clean": bool(
harness_source.get("worktree_clean") is True harness_source.get("worktree_clean") is True and _is_sha256(harness_source.get("status_sha256"))
and _is_sha256(harness_source.get("status_sha256"))
), ),
"actual_model_call": bool( "actual_model_call": bool(
model_execution.get("calls") model_execution.get("calls")
@ -801,11 +787,7 @@ def validate_turn_manifest(manifest: dict[str, Any]) -> list[str]:
problems = [] problems = []
if manifest.get("schema") != SCHEMA: if manifest.get("schema") != SCHEMA:
problems.append("schema_mismatch") problems.append("schema_mismatch")
stable = { stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
key: value
for key, value in manifest.items()
if key not in {"generated_at_utc", "execution_sha256"}
}
if manifest.get("execution_sha256") != canonical_sha256(stable): if manifest.get("execution_sha256") != canonical_sha256(stable):
problems.append("execution_sha256_mismatch") problems.append("execution_sha256_mismatch")
missing = (manifest.get("attribution") or {}).get("missing_required_bindings") missing = (manifest.get("attribution") or {}).get("missing_required_bindings")

15
scripts/run_leo_direct_claim_handler_suite.py
View file

@ -55,13 +55,11 @@ ASSIGNMENT_SECRET_PATTERN = re.compile(
re.IGNORECASE, re.IGNORECASE,
) )
DB_CONTEXT_PLUGIN = ( DB_CONTEXT_PLUGIN = ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py"
ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py"
)
KB_TOOL = ROOT / "hermes-agent" / "leoclean-bin" / "kb_tool.py" KB_TOOL = ROOT / "hermes-agent" / "leoclean-bin" / "kb_tool.py"
BEHAVIOR_MANIFEST = ROOT / "scripts" / "leo_behavior_manifest.py" BEHAVIOR_MANIFEST = ROOT / "scripts" / "leo_behavior_manifest.py"
RESPONSE_BINDING_HELPER_SOURCE = r''' RESPONSE_BINDING_HELPER_SOURCE = r"""
def response_bindings_are_hash_bound(results, records): def response_bindings_are_hash_bound(results, records):
if not isinstance(results, list) or not isinstance(records, list): if not isinstance(results, list) or not isinstance(records, list):
return False return False
@ -94,7 +92,7 @@ def response_bindings_are_hash_bound(results, records):
and valid_sha256(item.get("response_sha256")) and valid_sha256(item.get("response_sha256"))
for item in record_rows for item in record_rows
) )
''' """
REMOTE_SCRIPT = r''' REMOTE_SCRIPT = r'''
@ -1010,9 +1008,7 @@ def build_safety_gate(report: dict[str, Any]) -> dict[str, Any]:
handler = report.get("handler") if isinstance(report.get("handler"), dict) else {} handler = report.get("handler") if isinstance(report.get("handler"), dict) else {}
service = report.get("service_before_after") if isinstance(report.get("service_before_after"), dict) else {} service = report.get("service_before_after") if isinstance(report.get("service_before_after"), dict) else {}
execution_summary = ( execution_summary = (
report.get("execution_manifest_summary") report.get("execution_manifest_summary") if isinstance(report.get("execution_manifest_summary"), dict) else {}
if isinstance(report.get("execution_manifest_summary"), dict)
else {}
) )
checks = { checks = {
"remote_command_succeeded": report.get("remote_returncode") == 0, "remote_command_succeeded": report.get("remote_returncode") == 0,
@ -1020,8 +1016,7 @@ def build_safety_gate(report: dict[str, Any]) -> dict[str, Any]:
"handler_authorized": handler.get("authorized") is True, "handler_authorized": handler.get("authorized") is True,
"results_nonempty": bool(results), "results_nonempty": bool(results),
"all_turns_returned_replies": bool(results) and all(item.get("ok") is True for item in results), "all_turns_returned_replies": bool(results) and all(item.get("ok") is True for item in results),
"all_turns_declared_no_mutation": bool(results) "all_turns_declared_no_mutation": bool(results) and all(item.get("mutates_kb") is False for item in results),
and all(item.get("mutates_kb") is False for item in results),
"all_tool_traces_read_only_and_bound": bool(results) "all_tool_traces_read_only_and_bound": bool(results)
and all(_tool_trace_is_safe(item.get("database_tool_trace")) for item in results), and all(_tool_trace_is_safe(item.get("database_tool_trace")) for item in results),
"no_telegram_post": report.get("posted_to_telegram") is False, "no_telegram_post": report.get("posted_to_telegram") is False,

78
scripts/run_leo_m3taversal_oos_handler_suite.py
View file

@ -25,7 +25,6 @@ RESULTS_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-current.json"
SCORE_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.json" SCORE_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.json"
SCORE_MARKDOWN = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.md" SCORE_MARKDOWN = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.md"
PROTOCOL_REPORT_DIR = ROOT / "docs" / "reports" / "leo-oos-reasoning-benchmark-20260715" PROTOCOL_REPORT_DIR = ROOT / "docs" / "reports" / "leo-oos-reasoning-benchmark-20260715"
DEFAULT_PROTOCOL_JSON = PROTOCOL_REPORT_DIR / "blinded-protocol.json"
GROUNDING_MODES = ("grounded", "db_tool_ablated") GROUNDING_MODES = ("grounded", "db_tool_ablated")
RUNTIME_PROMPT_SCAN_ROOTS = ( RUNTIME_PROMPT_SCAN_ROOTS = (
ROOT / "hermes-agent" / "leoclean-skills", ROOT / "hermes-agent" / "leoclean-skills",
@ -45,27 +44,76 @@ def prompt_leakage_scan(protocol: dict[str, Any]) -> dict[str, Any]:
] ]
matches: list[dict[str, str]] = [] matches: list[dict[str, str]] = []
scanned_files = 0 scanned_files = 0
scanned_bytes = 0
scan_errors: list[dict[str, str]] = []
symlink_entries: list[dict[str, str]] = []
expected_roots = [
{
"path": str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path),
"exists": path.exists(),
"is_symlink": path.is_symlink(),
}
for path in RUNTIME_PROMPT_SCAN_ROOTS
]
for root in RUNTIME_PROMPT_SCAN_ROOTS: for root in RUNTIME_PROMPT_SCAN_ROOTS:
if not root.exists(): if not root.exists() or root.is_symlink():
continue continue
for path in sorted(item for item in root.rglob("*") if item.is_file()): try:
entries = sorted(root.rglob("*"))
except OSError as exc:
scan_errors.append(
{
"path_sha256": hashlib.sha256(str(root).encode()).hexdigest(),
"error_type": type(exc).__name__,
}
)
continue
for path in entries:
if path.is_symlink():
symlink_entries.append({"path_sha256": hashlib.sha256(str(path).encode()).hexdigest()})
paths = [item for item in entries if item.is_file() and not item.is_symlink()]
for path in paths:
try: try:
text = path.read_text(encoding="utf-8") text = path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError): except UnicodeDecodeError:
continue
except OSError as exc:
scan_errors.append(
{
"path_sha256": hashlib.sha256(str(path).encode()).hexdigest(),
"error_type": type(exc).__name__,
}
)
continue continue
scanned_files += 1 scanned_files += 1
scanned_bytes += len(text.encode())
normalized = " ".join(re.findall(r"[a-z0-9_]+", text.lower())) normalized = " ".join(re.findall(r"[a-z0-9_]+", text.lower()))
for prompt_id, marker in prompt_markers: for prompt_id, marker in prompt_markers:
if marker in normalized: if marker in normalized:
display_path = str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path) display_path = str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path)
matches.append( matches.append(
{"prompt_id": prompt_id, "marker_sha256": hashlib.sha256(marker.encode()).hexdigest(), "path": display_path} {
"prompt_id": prompt_id,
"marker_sha256": hashlib.sha256(marker.encode()).hexdigest(),
"path": display_path,
}
) )
return { return {
"scanned_roots": [str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path) for path in RUNTIME_PROMPT_SCAN_ROOTS], "expected_roots": expected_roots,
"scanned_roots": [item["path"] for item in expected_roots],
"scanned_files": scanned_files, "scanned_files": scanned_files,
"scanned_bytes": scanned_bytes,
"errors": scan_errors,
"symlink_entries": symlink_entries,
"exact_prompt_matches": matches, "exact_prompt_matches": matches,
"pass": not matches, "pass": bool(
all(item["exists"] and not item["is_symlink"] for item in expected_roots)
and scanned_files > 0
and scanned_bytes > 0
and not scan_errors
and not symlink_entries
and not matches
),
} }
@ -101,14 +149,14 @@ def build_guarded_remote_script(
1, 1,
) )
tool_trace_source = (ROOT / "scripts" / "leo_tool_trace.py").read_text(encoding="utf-8") tool_trace_source = (ROOT / "scripts" / "leo_tool_trace.py").read_text(encoding="utf-8")
remote_tool_trace_import = ''' sys.path.insert(0, str(DEPLOY_ROOT / "scripts")) remote_tool_trace_import = """ sys.path.insert(0, str(DEPLOY_ROOT / "scripts"))
from leo_tool_trace import extract_kb_tool_trace from leo_tool_trace import extract_kb_tool_trace
''' """
embedded_tool_trace_import = ''' tool_trace_module = types.ModuleType("leo_tool_trace_harness") embedded_tool_trace_import = """ tool_trace_module = types.ModuleType("leo_tool_trace_harness")
tool_trace_module.__file__ = "<embedded leo_tool_trace.py>" tool_trace_module.__file__ = "<embedded leo_tool_trace.py>"
exec(compile(LEO_TOOL_TRACE_SOURCE, tool_trace_module.__file__, "exec"), tool_trace_module.__dict__) exec(compile(LEO_TOOL_TRACE_SOURCE, tool_trace_module.__file__, "exec"), tool_trace_module.__dict__)
extract_kb_tool_trace = tool_trace_module.extract_kb_tool_trace extract_kb_tool_trace = tool_trace_module.extract_kb_tool_trace
''' """
if script.count(remote_tool_trace_import) != 1: if script.count(remote_tool_trace_import) != 1:
raise RuntimeError("remote leo_tool_trace import marker changed") raise RuntimeError("remote leo_tool_trace import marker changed")
script = script.replace(remote_tool_trace_import, embedded_tool_trace_import, 1) script = script.replace(remote_tool_trace_import, embedded_tool_trace_import, 1)
@ -758,7 +806,7 @@ def collect_restart_receipt(args: argparse.Namespace) -> int:
restart = _ssh_readback( restart = _ssh_readback(
"systemctl restart leoclean-gateway.service && " "systemctl restart leoclean-gateway.service && "
"for i in $(seq 1 60); do " "for i in $(seq 1 60); do "
"if [ \"$(systemctl is-active leoclean-gateway.service)\" = active ]; then break; fi; sleep 1; done; " 'if [ "$(systemctl is-active leoclean-gateway.service)" = active ]; then break; fi; sleep 1; done; '
"systemctl show leoclean-gateway.service " "systemctl show leoclean-gateway.service "
"-p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp", "-p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp",
timeout=120, timeout=120,
@ -803,8 +851,7 @@ def collect_restart_receipt(args: argparse.Namespace) -> int:
"db_fingerprint_unchanged": bool( "db_fingerprint_unchanged": bool(
fingerprint_before.get("status") == "ok" fingerprint_before.get("status") == "ok"
and fingerprint_after.get("status") == "ok" and fingerprint_after.get("status") == "ok"
and fingerprint_before.get("fingerprint_sha256") and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256")
== fingerprint_after.get("fingerprint_sha256")
), ),
"posted_to_telegram": False, "posted_to_telegram": False,
"before_probe": { "before_probe": {
@ -898,8 +945,7 @@ def run_or_score_protocol_trial(args: argparse.Namespace) -> int:
) )
mode_checks = { mode_checks = {
"expected_grounding_mode": report.get("grounding_mode") == args.grounding_mode, "expected_grounding_mode": report.get("grounding_mode") == args.grounding_mode,
"db_context_state": report.get("db_context_plugin_enabled") "db_context_state": report.get("db_context_plugin_enabled") is (args.grounding_mode == "grounded"),
is (args.grounding_mode == "grounded"),
"tool_surface_mode": (report.get("read_only_tool_surface") or {}).get("mode") "tool_surface_mode": (report.get("read_only_tool_surface") or {}).get("mode")
== ("read_only_kb" if args.grounding_mode == "grounded" else "no_db_ablation"), == ("read_only_kb" if args.grounding_mode == "grounded" else "no_db_ablation"),
"zero_context_in_ablation": args.grounding_mode != "db_tool_ablated" "zero_context_in_ablation": args.grounding_mode != "db_tool_ablated"

22
scripts/verify_leo_db_first_oos_canary.py
View file

@ -26,9 +26,7 @@ EXPECTED_SOURCE_IDS = frozenset(
) )
EXPECTED_SUBCOMMANDS = frozenset({"search", "show", "evidence"}) EXPECTED_SUBCOMMANDS = frozenset({"search", "show", "evidence"})
SHA_RE = re.compile(r"^[0-9a-f]{40}$") SHA_RE = re.compile(r"^[0-9a-f]{40}$")
UUID_RE = re.compile( UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b")
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b"
)
WORD_RE = re.compile(r"\b\w+(?:[-']\w+)*\b") WORD_RE = re.compile(r"\b\w+(?:[-']\w+)*\b")
@ -100,13 +98,10 @@ def verify_report(
and completed_without_error and completed_without_error
), ),
"database_tools_read_only": ( "database_tools_read_only": (
trace.get("database_tool_calls_read_only") is True trace.get("database_tool_calls_read_only") is True and trace.get("access_modes") == ["read_only"]
and trace.get("access_modes") == ["read_only"]
), ),
"retrieval_receipt_proven": trace.get("database_retrieval_receipt_proven") is True, "retrieval_receipt_proven": trace.get("database_retrieval_receipt_proven") is True,
"claim_and_sources_returned": ( "claim_and_sources_returned": (EXPECTED_CLAIM_ID in row_ids and row_ids >= EXPECTED_SOURCE_IDS),
EXPECTED_CLAIM_ID in row_ids and row_ids >= EXPECTED_SOURCE_IDS
),
"claim_support_challenged": _contains_all( "claim_support_challenged": _contains_all(
reply, reply,
("no_source_pointer", "grounds", "verified artifact", "illustrates"), ("no_source_pointer", "grounds", "verified artifact", "illustrates"),
@ -132,8 +127,7 @@ def verify_report(
and post_record.get("validated") is True and post_record.get("validated") is True
), ),
"canonical_counts_unchanged": ( "canonical_counts_unchanged": (
report.get("db_counts_changed") is False report.get("db_counts_changed") is False and report.get("db_counts_before") == report.get("db_counts_after")
and report.get("db_counts_before") == report.get("db_counts_after")
), ),
"canonical_content_fingerprint_unchanged": ( "canonical_content_fingerprint_unchanged": (
report.get("db_fingerprint_unchanged") is True report.get("db_fingerprint_unchanged") is True
@ -159,10 +153,7 @@ def verify_report(
and service_after.get("NRestarts") == "0" and service_after.get("NRestarts") == "0"
), ),
"temporary_profile_removed": report.get("temp_profile_removed") is True, "temporary_profile_removed": report.get("temp_profile_removed") is True,
"deploy_head_matches_stamp": ( "deploy_head_matches_stamp": (bool(SHA_RE.fullmatch(deploy_head)) and deploy_head == deploy_stamp),
bool(SHA_RE.fullmatch(deploy_head))
and deploy_head == deploy_stamp
),
} }
outcomes = { outcomes = {
"found_the_relevant_claim_without_a_supplied_id": ( "found_the_relevant_claim_without_a_supplied_id": (
@ -178,8 +169,7 @@ def verify_report(
checks["live_behavior_profile_unchanged"] and checks["temporary_profile_removed"] checks["live_behavior_profile_unchanged"] and checks["temporary_profile_removed"]
), ),
"did_not_change_canonical_knowledge": ( "did_not_change_canonical_knowledge": (
checks["canonical_counts_unchanged"] checks["canonical_counts_unchanged"] and checks["canonical_content_fingerprint_unchanged"]
and checks["canonical_content_fingerprint_unchanged"]
), ),
} }
passed = all(checks.values()) and all(outcomes.values()) passed = all(checks.values()) and all(outcomes.values())

229
scripts/working_leo_m3taversal_oos_protocol.py
View file

@ -390,19 +390,19 @@ def harness_git_head() -> str:
def instrument_db_context_plugin_source(source: str) -> str: def instrument_db_context_plugin_source(source: str) -> str:
marker = ''' safe["receipt_sha256"] = hashlib.sha256( marker = """ safe["receipt_sha256"] = hashlib.sha256(
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest() ).hexdigest()
return safe return safe
''' """
replacement = ''' safe["trace_payload_sha256"] = hashlib.sha256( replacement = """ safe["trace_payload_sha256"] = hashlib.sha256(
json.dumps(safe, sort_keys=True, separators=(",", ":")).encode("utf-8") json.dumps(safe, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest() ).hexdigest()
safe["receipt_sha256"] = hashlib.sha256( safe["receipt_sha256"] = hashlib.sha256(
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest() ).hexdigest()
return safe return safe
''' """
if source.count(marker) != 1: if source.count(marker) != 1:
raise RuntimeError("DB context receipt trace marker changed") raise RuntimeError("DB context receipt trace marker changed")
return source.replace(marker, replacement) return source.replace(marker, replacement)
@ -429,7 +429,12 @@ def source_paths() -> dict[str, Path]:
"execution_manifest_sha256": scripts / "leo_turn_execution_manifest.py", "execution_manifest_sha256": scripts / "leo_turn_execution_manifest.py",
"behavior_manifest_sha256": scripts / "leo_behavior_manifest.py", "behavior_manifest_sha256": scripts / "leo_behavior_manifest.py",
"tool_trace_sha256": scripts / "leo_tool_trace.py", "tool_trace_sha256": scripts / "leo_tool_trace.py",
"db_context_plugin_sha256": root / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py", "db_context_plugin_sha256": root
/ "hermes-agent"
/ "leoclean-plugins"
/ "vps"
/ "leo-db-context"
/ "__init__.py",
"db_context_plugin_manifest_sha256": root "db_context_plugin_manifest_sha256": root
/ "hermes-agent" / "hermes-agent"
/ "leoclean-plugins" / "leoclean-plugins"
@ -641,9 +646,7 @@ def validate_protocol(protocol: dict[str, Any], *, verify_source_hashes: bool) -
if requires_grounded_answer != (prompt.get("family_id") in AUTONOMOUS_RETRIEVAL_FAMILIES): if requires_grounded_answer != (prompt.get("family_id") in AUTONOMOUS_RETRIEVAL_FAMILIES):
issues.append(f"grounded_retrieval_answer_requirement_mismatch:{prompt_id}") issues.append(f"grounded_retrieval_answer_requirement_mismatch:{prompt_id}")
if requires_grounded_answer and ( if requires_grounded_answer and (
"teleo-kb context" in message "teleo-kb context" in message or "--limit" in message or "--context-limit" in message
or "--limit" in message
or "--context-limit" in message
): ):
issues.append(f"autonomous_retrieval_exact_command_leak:{prompt_id}") issues.append(f"autonomous_retrieval_exact_command_leak:{prompt_id}")
if requires_tool_evidence and ("teleo-kb context" not in message or "`Receipt:" not in message): if requires_tool_evidence and ("teleo-kb context" not in message or "`Receipt:" not in message):
@ -685,9 +688,7 @@ def _subject_alignment(prompt: dict[str, Any], reply: str) -> bool:
if str(item) != str(prompt.get("subject") or "") if str(item) != str(prompt.get("subject") or "")
} }
matches = { matches = {
str(anchor).lower() str(anchor).lower() for anchor in prompt.get("subject_anchors") or [] if str(anchor).lower() in reply.lower()
for anchor in prompt.get("subject_anchors") or []
if str(anchor).lower() in reply.lower()
} }
return ( return (
bool(normalized_subject) bool(normalized_subject)
@ -833,9 +834,7 @@ def _score_semantic_results(results: list[dict[str, Any]], trial: dict[str, Any]
} }
def _executed_behavior_ablation( def _executed_behavior_ablation(grounded_report: dict[str, Any], baseline_report: dict[str, Any]) -> dict[str, Any]:
grounded_report: dict[str, Any], baseline_report: dict[str, Any]
) -> dict[str, Any]:
grounded = grounded_report.get("executed_behavior_manifest") or {} grounded = grounded_report.get("executed_behavior_manifest") or {}
baseline = baseline_report.get("executed_behavior_manifest") or {} baseline = baseline_report.get("executed_behavior_manifest") or {}
stable_keys = { stable_keys = {
@ -884,30 +883,23 @@ def _executed_behavior_ablation(
and _valid_sha256(baseline.get("behavior_sha256")) and _valid_sha256(baseline.get("behavior_sha256"))
and baseline.get("behavior_sha256") == canonical_sha256(stable(baseline)), and baseline.get("behavior_sha256") == canonical_sha256(stable(baseline)),
"behavior_hashes_differ": grounded.get("behavior_sha256") != baseline.get("behavior_sha256"), "behavior_hashes_differ": grounded.get("behavior_sha256") != baseline.get("behavior_sha256"),
"component_sets_equal": bool(grounded_components) "component_sets_equal": bool(grounded_components) and set(grounded_components) == set(baseline_components),
and set(grounded_components) == set(baseline_components),
"non_middleware_components_equal": all( "non_middleware_components_equal": all(
grounded_components.get(name) == baseline_components.get(name) grounded_components.get(name) == baseline_components.get(name)
for name in set(grounded_components) | set(baseline_components) for name in set(grounded_components) | set(baseline_components)
if name != expected_component if name != expected_component
), ),
"top_level_runtime_equal": all( "top_level_runtime_equal": all(grounded.get(key) == baseline.get(key) for key in stable_keys - {"components"}),
grounded.get(key) == baseline.get(key) "middleware_metadata_equal": {key: value for key, value in grounded_middleware.items() if key != "content"}
for key in stable_keys - {"components"}
),
"middleware_metadata_equal": {
key: value for key, value in grounded_middleware.items() if key != "content"
}
== {key: value for key, value in baseline_middleware.items() if key != "content"}, == {key: value for key, value in baseline_middleware.items() if key != "content"},
"middleware_nonfile_state_equal": grounded_content.get("missing") == baseline_content.get("missing") "middleware_nonfile_state_equal": grounded_content.get("missing") == baseline_content.get("missing")
and grounded_content.get("symlinks") == baseline_content.get("symlinks"), and grounded_content.get("symlinks") == baseline_content.get("symlinks"),
"common_middleware_files_equal": bool(common_paths) "common_middleware_files_equal": bool(common_paths)
and all(grounded_files[path] == baseline_files[path] for path in common_paths), and all(grounded_files[path] == baseline_files[path] for path in common_paths),
"only_db_context_plugin_removed": extra_grounded_paths == expected_removed_paths "only_db_context_plugin_removed": extra_grounded_paths == expected_removed_paths and not extra_baseline_paths,
and not extra_baseline_paths, "grounded_db_context_source_is_exact_instrumented_source": grounded_files.get(expected_db_context_path, {}).get(
"grounded_db_context_source_is_exact_instrumented_source": grounded_files.get( "sha256"
expected_db_context_path, {} )
).get("sha256")
== instrumented_plugin_sha256, == instrumented_plugin_sha256,
"grounded_db_context_manifest_is_exact_frozen_source": grounded_files.get( "grounded_db_context_manifest_is_exact_frozen_source": grounded_files.get(
expected_db_context_manifest_path, {} expected_db_context_manifest_path, {}
@ -976,15 +968,11 @@ def _receipt_score(
) )
counts = receipt.get("counts") if isinstance(receipt.get("counts"), dict) else {} counts = receipt.get("counts") if isinstance(receipt.get("counts"), dict) else {}
receipt_counts_typed = all( receipt_counts_typed = all(
isinstance(counts.get(key), int) isinstance(counts.get(key), int) and not isinstance(counts.get(key), bool) and counts[key] >= 0
and not isinstance(counts.get(key), bool)
and counts[key] >= 0
for key in ("claims", "context_rows", "evidence_rows") for key in ("claims", "context_rows", "evidence_rows")
) )
safe_receipt_payload = { safe_receipt_payload = {
key: value key: value for key, value in receipt.items() if key not in {"receipt_sha256", "trace_payload_sha256"}
for key, value in receipt.items()
if key not in {"receipt_sha256", "trace_payload_sha256"}
} }
trace_payload_sha256 = hashlib.sha256( trace_payload_sha256 = hashlib.sha256(
json.dumps(safe_receipt_payload, sort_keys=True, separators=(",", ":")).encode("utf-8") json.dumps(safe_receipt_payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
@ -1000,10 +988,7 @@ def _receipt_score(
and typed_attempts and typed_attempts
and ( and (
(consistency.get("status") == "stable_wal_marker" and wal_before == wal_after) (consistency.get("status") == "stable_wal_marker" and wal_before == wal_after)
or ( or (consistency.get("status") == "stable_content_across_wal_change_retry" and attempts >= 2)
consistency.get("status") == "stable_content_across_wal_change_retry"
and attempts >= 2
)
) )
) )
if ( if (
@ -1017,8 +1002,7 @@ def _receipt_score(
and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("injected_rows_sha256") or "")) and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("injected_rows_sha256") or ""))
and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("receipt_sha256") or "")) and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("receipt_sha256") or ""))
and receipt.get("trace_payload_sha256") == trace_payload_sha256 and receipt.get("trace_payload_sha256") == trace_payload_sha256
and consistency.get("status") and consistency.get("status") in {"stable_wal_marker", "stable_content_across_wal_change_retry"}
in {"stable_wal_marker", "stable_content_across_wal_change_retry"}
and typed_attempts and typed_attempts
and consistency.get("database") and consistency.get("database")
and consistency.get("database_user") and consistency.get("database_user")
@ -1054,10 +1038,7 @@ def _receipt_score(
reply_claim_citations = {match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(reply)} reply_claim_citations = {match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(reply)}
reply_source_citations = {match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(reply)} reply_source_citations = {match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(reply)}
unsupported_identifiers = sorted(reply_identifiers - supported_identifiers) unsupported_identifiers = sorted(reply_identifiers - supported_identifiers)
receipt_counts = [ receipt_counts = [(item.get("retrieval_receipt") or {}).get("counts") or {} for item in retrieval_records]
(item.get("retrieval_receipt") or {}).get("counts") or {}
for item in retrieval_records
]
reply_sha256 = hashlib.sha256(reply.encode()).hexdigest() reply_sha256 = hashlib.sha256(reply.encode()).hexdigest()
checks = { checks = {
"reply_present": result.get("ok") is True and bool(str(result.get("reply") or "").strip()), "reply_present": result.get("ok") is True and bool(str(result.get("reply") or "").strip()),
@ -1073,8 +1054,7 @@ def _receipt_score(
"post_contract_satisfaction_reported": post_contract_satisfaction_reported, "post_contract_satisfaction_reported": post_contract_satisfaction_reported,
"contract_ids_are_typed_lists": contract_ids_are_lists, "contract_ids_are_typed_lists": contract_ids_are_lists,
"context_response_query_hash_bound": pre_hashes == post_hashes == {prompt_sha256}, "context_response_query_hash_bound": pre_hashes == post_hashes == {prompt_sha256},
"delivered_response_hash_bound": len(post) == 1 "delivered_response_hash_bound": len(post) == 1 and post[0].get("delivered_response_sha256") == reply_sha256,
and post[0].get("delivered_response_sha256") == reply_sha256,
"database_contract_present": bool(contract_ids - NON_DB_CONTRACT_IDS) if require_database_contract else True, "database_contract_present": bool(contract_ids - NON_DB_CONTRACT_IDS) if require_database_contract else True,
"database_retrieval_receipt_present": bool(retrieval_records) if require_database_receipt else True, "database_retrieval_receipt_present": bool(retrieval_records) if require_database_receipt else True,
"grounded_claim_rows_nonempty": any( "grounded_claim_rows_nonempty": any(
@ -1094,7 +1074,10 @@ def _receipt_score(
if require_grounded_rows if require_grounded_rows
else True, else True,
"model_call_receipt_present": bool(model_call_trace) "model_call_receipt_present": bool(model_call_trace)
and any(item.get("event") == "post_api_request" and item.get("model") and item.get("provider") for item in model_call_trace), and any(
item.get("event") == "post_api_request" and item.get("model") and item.get("provider")
for item in model_call_trace
),
"conversation_history_prefix_preserved": result.get("conversation_history_prefix_preserved") is True, "conversation_history_prefix_preserved": result.get("conversation_history_prefix_preserved") is True,
"no_unsupported_exact_identifiers": not unsupported_identifiers, "no_unsupported_exact_identifiers": not unsupported_identifiers,
} }
@ -1174,7 +1157,9 @@ def _benchmark_execution_chain(
suite_safety = delivery.get("suite_safety") or {} suite_safety = delivery.get("suite_safety") or {}
attribution = manifest.get("attribution") or {} attribution = manifest.get("attribution") or {}
missing = attribution.get("missing_required_bindings") missing = attribution.get("missing_required_bindings")
missing_set = set(missing) if isinstance(missing, list) and all(isinstance(item, str) for item in missing) else set() missing_set = (
set(missing) if isinstance(missing, list) and all(isinstance(item, str) for item in missing) else set()
)
hermes_runtime = runtime.get("hermes_runtime") or {} hermes_runtime = runtime.get("hermes_runtime") or {}
teleo_runtime = runtime.get("teleo_infrastructure_runtime") or {} teleo_runtime = runtime.get("teleo_infrastructure_runtime") or {}
calls = model.get("calls") if isinstance(model.get("calls"), list) else [] calls = model.get("calls") if isinstance(model.get("calls"), list) else []
@ -1184,8 +1169,7 @@ def _benchmark_execution_chain(
else [] else []
) )
checks = { checks = {
"generic_manifest_valid": bool(manifest) "generic_manifest_valid": bool(manifest) and not execution_manifest_lib.validate_turn_manifest(manifest),
and not execution_manifest_lib.validate_turn_manifest(manifest),
"prompt_bound": turn.get("prompt_id") == result.get("prompt_id") "prompt_bound": turn.get("prompt_id") == result.get("prompt_id")
and turn.get("prompt_sha256") == hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest(), and turn.get("prompt_sha256") == hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest(),
"reply_bound": turn.get("reply_sha256") "reply_bound": turn.get("reply_sha256")
@ -1304,9 +1288,7 @@ def _top_level_safety(
and all(value is True for key, value in handler_checks.items() if key != "all_turn_manifests_complete") and all(value is True for key, value in handler_checks.items() if key != "all_turn_manifests_complete")
) )
checks = { checks = {
"fresh_temporary_session": (report.get("temp_profile_seed") or {}).get( "fresh_temporary_session": (report.get("temp_profile_seed") or {}).get("same_session_continuity_starts_fresh")
"same_session_continuity_starts_fresh"
)
is True is True
and bool(result_rows) and bool(result_rows)
and (result_rows[0].get("conversation_before") or {}).get("message_count") == 0, and (result_rows[0].get("conversation_before") or {}).get("message_count") == 0,
@ -1328,8 +1310,7 @@ def _top_level_safety(
"database_fingerprint_after_ok": after.get("status") == "ok", "database_fingerprint_after_ok": after.get("status") == "ok",
"database_fingerprint_unchanged": report.get("db_fingerprint_unchanged") is True, "database_fingerprint_unchanged": report.get("db_fingerprint_unchanged") is True,
"database_fingerprint_hash_equal": bool( "database_fingerprint_hash_equal": bool(
before.get("fingerprint_sha256") before.get("fingerprint_sha256") and before.get("fingerprint_sha256") == after.get("fingerprint_sha256")
and before.get("fingerprint_sha256") == after.get("fingerprint_sha256")
), ),
"live_behavior_manifest_unchanged": report.get("live_behavior_manifest_unchanged") is True, "live_behavior_manifest_unchanged": report.get("live_behavior_manifest_unchanged") is True,
"service_unchanged_during_trial": service.get("unchanged_from_preexisting_live_readback") is True, "service_unchanged_during_trial": service.get("unchanged_from_preexisting_live_readback") is True,
@ -1407,9 +1388,9 @@ def _prompt_binding(report: dict[str, Any], trial: dict[str, Any]) -> dict[str,
for prompt_id, prompt in expected.items(): for prompt_id, prompt in expected.items():
result = actual.get(prompt_id) or {} result = actual.get(prompt_id) or {}
checks[f"prompt_text:{prompt_id}"] = result.get("prompt") == prompt["message"] checks[f"prompt_text:{prompt_id}"] = result.get("prompt") == prompt["message"]
checks[f"prompt_hash:{prompt_id}"] = hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest() == prompt[ checks[f"prompt_hash:{prompt_id}"] = (
"message_sha256" hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest() == prompt["message_sha256"]
] )
return {"checks": checks, "pass": all(checks.values())} return {"checks": checks, "pass": all(checks.values())}
@ -1434,7 +1415,9 @@ def _nonempty_integer_mapping(value: Any) -> bool:
return bool( return bool(
isinstance(value, dict) isinstance(value, dict)
and value and value
and all(isinstance(key, str) and isinstance(item, int) and not isinstance(item, bool) for key, item in value.items()) and all(
isinstance(key, str) and isinstance(item, int) and not isinstance(item, bool) for key, item in value.items()
)
) )
@ -1570,7 +1553,9 @@ def validate_restart_receipt(receipt: dict[str, Any] | None, report: dict[str, A
"restart_command_succeeded": receipt.get("restart_returncode") == 0, "restart_command_succeeded": receipt.get("restart_returncode") == 0,
"service_active_after": after.get("ActiveState") == "active" and after.get("SubState") == "running", "service_active_after": after.get("ActiveState") == "active" and after.get("SubState") == "running",
"service_pid_changed": bool(before.get("MainPID") and before.get("MainPID") != after.get("MainPID")), "service_pid_changed": bool(before.get("MainPID") and before.get("MainPID") != after.get("MainPID")),
"trial_observed_restarted_pid": bool(after.get("MainPID") and report_before.get("MainPID") == after.get("MainPID")), "trial_observed_restarted_pid": bool(
after.get("MainPID") and report_before.get("MainPID") == after.get("MainPID")
),
"service_start_identity_bound": bool( "service_start_identity_bound": bool(
after.get("ExecMainStartTimestamp") after.get("ExecMainStartTimestamp")
and after.get("ExecMainStartTimestamp") == report_before.get("ExecMainStartTimestamp") and after.get("ExecMainStartTimestamp") == report_before.get("ExecMainStartTimestamp")
@ -1676,7 +1661,12 @@ def score_live_trial(
"semantic_score": semantic_item, "semantic_score": semantic_item,
"receipt_score": receipt_item, "receipt_score": receipt_item,
"reply_sha256": hashlib.sha256( "reply_sha256": hashlib.sha256(
str((next((row for row in report_results if row.get("prompt_id") == prompt_id), {}) or {}).get("reply") or "").encode() str(
(next((row for row in report_results if row.get("prompt_id") == prompt_id), {}) or {}).get(
"reply"
)
or ""
).encode()
).hexdigest(), ).hexdigest(),
} }
) )
@ -1698,7 +1688,9 @@ def score_live_trial(
baseline_receipts = { baseline_receipts = {
str(result.get("prompt_id")): _receipt_score( str(result.get("prompt_id")): _receipt_score(
result, result,
require_database_contract=bool(by_prompt.get(str(result.get("prompt_id")), {}).get("requires_database_contract")), require_database_contract=bool(
by_prompt.get(str(result.get("prompt_id")), {}).get("requires_database_contract")
),
require_database_receipt=bool( require_database_receipt=bool(
by_prompt.get(str(result.get("prompt_id")), {}).get("requires_database_receipt") by_prompt.get(str(result.get("prompt_id")), {}).get("requires_database_receipt")
), ),
@ -1729,9 +1721,7 @@ def score_live_trial(
and baseline_receipts.get(prompt["id"], {}).get("pass") and baseline_receipts.get(prompt["id"], {}).get("pass")
) )
autonomous_prompt_ids = [ autonomous_prompt_ids = [
prompt["id"] prompt["id"] for prompt in trial["prompts"] if prompt.get("requires_grounded_retrieval_answer") is True
for prompt in trial["prompts"]
if prompt.get("requires_grounded_retrieval_answer") is True
] ]
autonomous_rows: list[dict[str, Any]] = [] autonomous_rows: list[dict[str, Any]] = []
for prompt_id in autonomous_prompt_ids: for prompt_id in autonomous_prompt_ids:
@ -1741,15 +1731,9 @@ def score_live_trial(
supported_claim_ids = set(grounded_receipt_score.get("supported_claim_ids") or []) supported_claim_ids = set(grounded_receipt_score.get("supported_claim_ids") or [])
supported_source_ids = set(grounded_receipt_score.get("supported_source_ids") or []) supported_source_ids = set(grounded_receipt_score.get("supported_source_ids") or [])
supported_identifiers = set(grounded_receipt_score.get("supported_identifiers") or []) supported_identifiers = set(grounded_receipt_score.get("supported_identifiers") or [])
ablation_claim_citations = { ablation_claim_citations = {match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(baseline_reply)}
match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(baseline_reply) ablation_source_citations = {match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(baseline_reply)}
} ablation_reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(baseline_reply)}
ablation_source_citations = {
match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(baseline_reply)
}
ablation_reply_identifiers = {
match.group(0).lower() for match in UUID_RE.finditer(baseline_reply)
}
grounded_answer_pass = bool( grounded_answer_pass = bool(
semantic_by_prompt.get(prompt_id, {}).get("pass") semantic_by_prompt.get(prompt_id, {}).get("pass")
and subject_alignment.get(prompt_id) and subject_alignment.get(prompt_id)
@ -1782,9 +1766,7 @@ def score_live_trial(
autonomous_prompt_count = len(autonomous_rows) autonomous_prompt_count = len(autonomous_rows)
autonomous_grounded_passes = sum(1 for row in autonomous_rows if row["grounded_answer_pass"]) autonomous_grounded_passes = sum(1 for row in autonomous_rows if row["grounded_answer_pass"])
autonomous_ablation_passes = sum(1 for row in autonomous_rows if row["ablation_answer_pass"]) autonomous_ablation_passes = sum(1 for row in autonomous_rows if row["ablation_answer_pass"])
autonomous_causal_passes = sum( autonomous_causal_passes = sum(1 for row in autonomous_rows if row["causally_attributed_grounded_pass"])
1 for row in autonomous_rows if row["causally_attributed_grounded_pass"]
)
autonomous_retrieval_comparison = { autonomous_retrieval_comparison = {
"method": "both_arms_semantic_subject_and_citations_scored_against_grounded_receipted_ids", "method": "both_arms_semantic_subject_and_citations_scored_against_grounded_receipted_ids",
"prompt_ids": autonomous_prompt_ids, "prompt_ids": autonomous_prompt_ids,
@ -1797,9 +1779,7 @@ def score_live_trial(
if autonomous_prompt_count if autonomous_prompt_count
else 0.0 else 0.0
), ),
"identical_reply_prompt_ids": [ "identical_reply_prompt_ids": [row["prompt_id"] for row in autonomous_rows if row["replies_identical"]],
row["prompt_id"] for row in autonomous_rows if row["replies_identical"]
],
"rows": autonomous_rows, "rows": autonomous_rows,
"pass": bool( "pass": bool(
autonomous_prompt_count autonomous_prompt_count
@ -1812,8 +1792,7 @@ def score_live_trial(
prompt["id"] for prompt in trial["prompts"] if prompt.get("requires_tool_evidence_token") is True prompt["id"] for prompt in trial["prompts"] if prompt.get("requires_tool_evidence_token") is True
] ]
grounded_evidence_by_prompt = { grounded_evidence_by_prompt = {
item["prompt_id"]: item["evidence_answer_score"]["grounded_tool_semantic_hashes"] item["prompt_id"]: item["evidence_answer_score"]["grounded_tool_semantic_hashes"] for item in prompt_scores
for item in prompt_scores
} }
baseline_evidence_scores = { baseline_evidence_scores = {
prompt_id: _evidence_answer_score( prompt_id: _evidence_answer_score(
@ -1826,9 +1805,7 @@ def score_live_trial(
for prompt_id in evidence_prompt_ids for prompt_id in evidence_prompt_ids
} }
current_evidence_passes = sum( current_evidence_passes = sum(
1 1 for item in prompt_scores if item["prompt_id"] in evidence_prompt_ids and item["evidence_answer_pass"] is True
for item in prompt_scores
if item["prompt_id"] in evidence_prompt_ids and item["evidence_answer_pass"] is True
) )
baseline_evidence_passes = sum(1 for item in baseline_evidence_scores.values() if item["pass"] is True) baseline_evidence_passes = sum(1 for item in baseline_evidence_scores.values() if item["pass"] is True)
evidence_prompt_count = len(evidence_prompt_ids) evidence_prompt_count = len(evidence_prompt_ids)
@ -1858,6 +1835,7 @@ def score_live_trial(
not item["receipt_score"].get("unsupported_identifiers") for item in prompt_scores not item["receipt_score"].get("unsupported_identifiers") for item in prompt_scores
), ),
} }
def model_identities(value: dict[str, Any]) -> list[dict[str, Any]]: def model_identities(value: dict[str, Any]) -> list[dict[str, Any]]:
identities = { identities = {
canonical_sha256( canonical_sha256(
@ -1889,7 +1867,9 @@ def score_live_trial(
"prompt_set_hash_equal": baseline_report.get("trial_prompt_set_sha256") "prompt_set_hash_equal": baseline_report.get("trial_prompt_set_sha256")
== report.get("trial_prompt_set_sha256") == report.get("trial_prompt_set_sha256")
== trial["prompt_set_sha256"], == trial["prompt_set_sha256"],
"source_hashes_equal": baseline_report.get("source_hashes") == report.get("source_hashes") == protocol["source_hashes"], "source_hashes_equal": baseline_report.get("source_hashes")
== report.get("source_hashes")
== protocol["source_hashes"],
"live_behavior_manifest_equal": bool( "live_behavior_manifest_equal": bool(
(baseline_report.get("live_behavior_manifest_before") or {}).get("behavior_sha256") (baseline_report.get("live_behavior_manifest_before") or {}).get("behavior_sha256")
and (baseline_report.get("live_behavior_manifest_before") or {}).get("behavior_sha256") and (baseline_report.get("live_behavior_manifest_before") or {}).get("behavior_sha256")
@ -1911,9 +1891,7 @@ def score_live_trial(
} }
threshold = float(protocol["thresholds"]["minimum_trial_grounded_pass_rate"]) threshold = float(protocol["thresholds"]["minimum_trial_grounded_pass_rate"])
evidence_threshold = float(protocol["thresholds"]["minimum_trial_evidence_answer_pass_rate"]) evidence_threshold = float(protocol["thresholds"]["minimum_trial_evidence_answer_pass_rate"])
evidence_delta_threshold = float( evidence_delta_threshold = float(protocol["thresholds"]["minimum_current_minus_ablation_evidence_answer_delta"])
protocol["thresholds"]["minimum_current_minus_ablation_evidence_answer_delta"]
)
score: dict[str, Any] = { score: dict[str, Any] = {
"schema": TRIAL_SCORE_SCHEMA, "schema": TRIAL_SCORE_SCHEMA,
"generated_at_utc": datetime.now(timezone.utc).isoformat(), "generated_at_utc": datetime.now(timezone.utc).isoformat(),
@ -2059,8 +2037,7 @@ def validate_trial_score(protocol: dict[str, Any], trial: dict[str, Any], score:
"protocol_id_bound": payload.get("protocol_id") == protocol.get("protocol_id"), "protocol_id_bound": payload.get("protocol_id") == protocol.get("protocol_id"),
"protocol_hash_bound": payload.get("protocol_hash_sha256") == protocol.get("protocol_hash_sha256"), "protocol_hash_bound": payload.get("protocol_hash_sha256") == protocol.get("protocol_hash_sha256"),
"next_trial_id_bound": payload.get("next_trial_id") == trial.get("trial_id"), "next_trial_id_bound": payload.get("next_trial_id") == trial.get("trial_id"),
"next_prompt_set_bound": payload.get("next_trial_prompt_set_sha256") "next_prompt_set_bound": payload.get("next_trial_prompt_set_sha256") == trial.get("prompt_set_sha256"),
== trial.get("prompt_set_sha256"),
}, payload }, payload
expected_prompt_ids = [item["id"] for item in trial["prompts"]] expected_prompt_ids = [item["id"] for item in trial["prompts"]]
@ -2076,9 +2053,7 @@ def validate_trial_score(protocol: dict[str, Any], trial: dict[str, Any], score:
baseline_semantic_scores = ( baseline_semantic_scores = (
baseline.get("semantic_scores") if isinstance(baseline.get("semantic_scores"), list) else [] baseline.get("semantic_scores") if isinstance(baseline.get("semantic_scores"), list) else []
) )
baseline_semantic_ids = [ baseline_semantic_ids = [str(item.get("prompt_id")) for item in baseline_semantic_scores if isinstance(item, dict)]
str(item.get("prompt_id")) for item in baseline_semantic_scores if isinstance(item, dict)
]
baseline_semantic_passes = sum( baseline_semantic_passes = sum(
1 for item in baseline_semantic_scores if isinstance(item, dict) and item.get("pass") is True 1 for item in baseline_semantic_scores if isinstance(item, dict) and item.get("pass") is True
) )
@ -2160,25 +2135,17 @@ def validate_trial_score(protocol: dict[str, Any], trial: dict[str, Any], score:
"evidence_prompt_count": evidence.get("prompt_count") == evidence_count, "evidence_prompt_count": evidence.get("prompt_count") == evidence_count,
"current_evidence_passes_recomputed": evidence.get("current_passes") == current_evidence_passes, "current_evidence_passes_recomputed": evidence.get("current_passes") == current_evidence_passes,
"baseline_evidence_passes_recomputed": evidence.get("ablation_passes") == baseline_evidence_passes, "baseline_evidence_passes_recomputed": evidence.get("ablation_passes") == baseline_evidence_passes,
"current_evidence_rate_recomputed": abs( "current_evidence_rate_recomputed": abs(number(evidence.get("current_pass_rate")) - current_evidence_rate)
number(evidence.get("current_pass_rate")) - current_evidence_rate
)
< 1e-12, < 1e-12,
"baseline_evidence_rate_recomputed": abs( "baseline_evidence_rate_recomputed": abs(number(evidence.get("ablation_pass_rate")) - baseline_evidence_rate)
number(evidence.get("ablation_pass_rate")) - baseline_evidence_rate
)
< 1e-12, < 1e-12,
"evidence_delta_recomputed": abs( "evidence_delta_recomputed": abs(
number(evidence.get("current_minus_ablation_delta")) number(evidence.get("current_minus_ablation_delta")) - (current_evidence_rate - baseline_evidence_rate)
- (current_evidence_rate - baseline_evidence_rate)
) )
< 1e-12, < 1e-12,
"grounded_report_artifact_bound": bool(grounded_artifact_checks) "grounded_report_artifact_bound": bool(grounded_artifact_checks) and all(grounded_artifact_checks.values()),
and all(grounded_artifact_checks.values()), "baseline_report_artifact_bound": bool(baseline_artifact_checks) and all(baseline_artifact_checks.values()),
"baseline_report_artifact_bound": bool(baseline_artifact_checks) "restart_receipt_artifact_bound": bool(restart_artifact_checks) and all(restart_artifact_checks.values()),
and all(baseline_artifact_checks.values()),
"restart_receipt_artifact_bound": bool(restart_artifact_checks)
and all(restart_artifact_checks.values()),
"derivation_core_hash_bound": _valid_sha256(score.get("derivation_core_sha256")) "derivation_core_hash_bound": _valid_sha256(score.get("derivation_core_sha256"))
and score.get("derivation_core_sha256") == canonical_sha256(stored_core), and score.get("derivation_core_sha256") == canonical_sha256(stored_core),
"score_recomputed_from_retained_artifacts": bool(recomputed) "score_recomputed_from_retained_artifacts": bool(recomputed)
@ -2197,9 +2164,7 @@ def validate_trial_score(protocol: dict[str, Any], trial: dict[str, Any], score:
and all(value is True for value in mapping(score.get("comparison_axis_checks")).values()), and all(value is True for value in mapping(score.get("comparison_axis_checks")).values()),
"critical_prompt_checks_passed": bool(mapping(score.get("critical_prompt_checks"))) "critical_prompt_checks_passed": bool(mapping(score.get("critical_prompt_checks")))
and all(value is True for value in mapping(score.get("critical_prompt_checks")).values()), and all(value is True for value in mapping(score.get("critical_prompt_checks")).values()),
"autonomous_retrieval_comparison_passed": mapping( "autonomous_retrieval_comparison_passed": mapping(score.get("autonomous_retrieval_comparison")).get("pass")
score.get("autonomous_retrieval_comparison")
).get("pass")
is True, is True,
"restart_receipt_validation_passed": mapping(score.get("restart_receipt_validation")).get("pass") is True, "restart_receipt_validation_passed": mapping(score.get("restart_receipt_validation")).get("pass") is True,
"score_passed": score.get("pass") is True, "score_passed": score.get("pass") is True,
@ -2222,7 +2187,9 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
trial_id: validate_trial_score(protocol, trial_by_id[trial_id], by_id.get(trial_id, {})) trial_id: validate_trial_score(protocol, trial_by_id[trial_id], by_id.get(trial_id, {}))
for trial_id in expected_ids for trial_id in expected_ids
} }
current_rates = [float(by_id[trial_id].get("grounded_pass_rate") or 0.0) for trial_id in expected_ids if trial_id in by_id] current_rates = [
float(by_id[trial_id].get("grounded_pass_rate") or 0.0) for trial_id in expected_ids if trial_id in by_id
]
baseline_rates = [ baseline_rates = [
float((by_id[trial_id].get("receipt_ablation") or {}).get("grounded_pass_rate") or 0.0) float((by_id[trial_id].get("receipt_ablation") or {}).get("grounded_pass_rate") or 0.0)
for trial_id in expected_ids for trial_id in expected_ids
@ -2251,9 +2218,7 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
] ]
autonomous_retrieval_deltas = [ autonomous_retrieval_deltas = [
float( float(
(by_id[trial_id].get("autonomous_retrieval_comparison") or {}).get( (by_id[trial_id].get("autonomous_retrieval_comparison") or {}).get("grounded_minus_ablation_answer_delta")
"grounded_minus_ablation_answer_delta"
)
or 0.0 or 0.0
) )
for trial_id in expected_ids for trial_id in expected_ids
@ -2285,8 +2250,7 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
"minimum_non_tautological_evidence_ablation_delta": (mean_evidence_current - mean_evidence_baseline) "minimum_non_tautological_evidence_ablation_delta": (mean_evidence_current - mean_evidence_baseline)
>= float(thresholds["minimum_current_minus_ablation_evidence_answer_delta"]), >= float(thresholds["minimum_current_minus_ablation_evidence_answer_delta"]),
"baseline_rejects_all_ungrounded_receipts": all(rate == 0.0 for rate in baseline_rates), "baseline_rejects_all_ungrounded_receipts": all(rate == 0.0 for rate in baseline_rates),
"autonomous_retrieval_has_causal_lift_every_trial": len(autonomous_retrieval_deltas) "autonomous_retrieval_has_causal_lift_every_trial": len(autonomous_retrieval_deltas) == len(expected_ids)
== len(expected_ids)
and all(delta == 1.0 for delta in autonomous_retrieval_deltas), and all(delta == 1.0 for delta in autonomous_retrieval_deltas),
"broad_semantic_comparison_reported": len(semantic_current_rates) "broad_semantic_comparison_reported": len(semantic_current_rates)
== len(semantic_baseline_rates) == len(semantic_baseline_rates)
@ -2297,6 +2261,16 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
if item["session_mode"] == "post_restart_clean_session" if item["session_mode"] == "post_restart_clean_session"
), ),
} }
evaluation_pass = all(checks.values())
independent_live_attestation = {
"status": "not_configured",
"pass": False,
"required_for": "machine_verifiable_T3_live_readonly",
"reason": (
"The protocol, reports, and scores are caller-readable artifacts. They validate benchmark semantics "
"but do not contain a non-caller-mintable platform or service attestation of live execution."
),
}
aggregate = { aggregate = {
"schema": AGGREGATE_SCHEMA, "schema": AGGREGATE_SCHEMA,
"generated_at_utc": datetime.now(timezone.utc).isoformat(), "generated_at_utc": datetime.now(timezone.utc).isoformat(),
@ -2318,9 +2292,7 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
"mean_ablation_evidence_answer_pass_rate": mean_evidence_baseline, "mean_ablation_evidence_answer_pass_rate": mean_evidence_baseline,
"current_minus_ablation_evidence_answer_delta": mean_evidence_current - mean_evidence_baseline, "current_minus_ablation_evidence_answer_delta": mean_evidence_current - mean_evidence_baseline,
"current_evidence_answer_pass_rate_population_stddev": evidence_stddev, "current_evidence_answer_pass_rate_population_stddev": evidence_stddev,
"minimum_current_evidence_answer_pass_rate": min(evidence_current_rates) "minimum_current_evidence_answer_pass_rate": min(evidence_current_rates) if evidence_current_rates else 0.0,
if evidence_current_rates
else 0.0,
"current_semantic_pass_rates": semantic_current_rates, "current_semantic_pass_rates": semantic_current_rates,
"ablation_semantic_pass_rates": semantic_baseline_rates, "ablation_semantic_pass_rates": semantic_baseline_rates,
"mean_current_semantic_pass_rate": mean_semantic_current, "mean_current_semantic_pass_rate": mean_semantic_current,
@ -2331,16 +2303,19 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
"trial_score_integrity": integrity, "trial_score_integrity": integrity,
"trial_scores": trial_scores, "trial_scores": trial_scores,
"required_tier": "T3_live_readonly", "required_tier": "T3_live_readonly",
"current_tier": "T2_runtime_artifact_validation",
"operator_observed_tier": "T3_live_readonly_unattested",
"evaluation_pass": evaluation_pass,
"independent_live_attestation": independent_live_attestation,
"machine_verifiable_t3_pass": evaluation_pass and independent_live_attestation["pass"],
"claim_ceiling": ( "claim_ceiling": (
"Repeated live VPS GatewayRunner reasoning with frozen blinded families and a current-build no-DB " "The retained artifacts can validate frozen-family scoring, grounded-versus-ablated comparisons, "
"ablation. Both arms are scored against the grounded arm's model-visible receipt tokens, so identical " "tool/response binding, and unchanged database fingerprints at T2. An operator may separately report "
"answers cannot create the gated evidence delta. Broad semantic rates for both arms are reported " "that they were captured from a live VPS, but these caller-readable files do not independently prove "
"separately and are descriptive rather than silently inferred from missing receipts. Complete " "that T3 origin. No Telegram delivery or production apply is proven."
"tool/execution receipts and unchanged canonical database fingerprints are retained. No Telegram "
"delivery or production apply is proven."
), ),
} }
aggregate["pass"] = all(checks.values()) aggregate["pass"] = aggregate["machine_verifiable_t3_pass"]
return aggregate return aggregate

13
tests/test_hermes_leoclean_db_context_plugin.py
View file

@ -121,7 +121,7 @@ def test_plugin_injects_bounded_retrieval_rows_and_writes_body_redacted_trace(tm
assert "NEVER_INCLUDE_RAW_UNUSED_FIELD" not in context assert "NEVER_INCLUDE_RAW_UNUSED_FIELD" not in context
assert len(context) < 8_000 assert len(context) < 8_000
injected_hash_match = re.search(r'injected_rows_sha256="([0-9a-f]{64})"', context) injected_hash_match = re.search(r'injected_rows_sha256="([0-9a-f]{64})"', context)
injected_payload_match = re.search(r'\n(\{[^\n]+\})\n</leo_retrieved_database_rows>', context) injected_payload_match = re.search(r"\n(\{[^\n]+\})\n</leo_retrieved_database_rows>", context)
assert injected_hash_match is not None assert injected_hash_match is not None
assert injected_payload_match is not None assert injected_payload_match is not None
injected_rows_sha256 = injected_hash_match.group(1) injected_rows_sha256 = injected_hash_match.group(1)
@ -275,9 +275,7 @@ def test_plugin_compacts_generic_over_budget_reply_without_a_query_template(tmp_
"What should we challenge first?" "What should we challenge first?"
) )
result = module._post_llm_call( result = module._post_llm_call(session_id="budget-session", user_message=query, assistant_response=draft)
session_id="budget-session", user_message=query, assistant_response=draft
)
assert result is not None assert result is not None
delivered = result["assistant_response"] delivered = result["assistant_response"]
@ -320,9 +318,7 @@ def test_broad_report_learning_question_requires_database_truth() -> None:
assert module._requires_database_truth(query) is True assert module._requires_database_truth(query) is True
def test_plugin_preserves_same_session_chat_only_memory_set_and_recall_responses( def test_plugin_preserves_same_session_chat_only_memory_set_and_recall_responses(tmp_path: Path, monkeypatch) -> None:
tmp_path: Path, monkeypatch
) -> None:
module = load_plugin() module = load_plugin()
trace = tmp_path / "trace.jsonl" trace = tmp_path / "trace.jsonl"
monkeypatch.setenv("LEO_DB_CONTEXT_TRACE_PATH", str(trace)) monkeypatch.setenv("LEO_DB_CONTEXT_TRACE_PATH", str(trace))
@ -478,8 +474,7 @@ def test_plugin_requires_named_proposal_receipt_when_live_match_exists() -> None
valid = ( valid = (
"No.\n" "No.\n"
"DB readback: proposal: 10bc0719-1c2b-5f42-a41e-86e6478692cb; status: pending_review; " "DB readback: proposal: 10bc0719-1c2b-5f42-a41e-86e6478692cb; status: pending_review; "
"applied_at: none; readiness: needs_human_review.\n" "applied_at: none; readiness: needs_human_review.\n" + common
+ common
) )
missing_receipt_issues = module.response_contract_issues("No.\n" + common, contracts) missing_receipt_issues = module.response_contract_issues("No.\n" + common, contracts)

4
tests/test_leo_tool_trace.py
View file

@ -25,7 +25,7 @@ def test_extracts_completed_read_only_context_call_and_receipt() -> None:
"id": "call-secret-looking-id", "id": "call-secret-looking-id",
"function": { "function": {
"name": "terminal", "name": "terminal",
"arguments": '{"command":"teleo-kb context \\\"AI sandbagging\\\""}', "arguments": '{"command":"teleo-kb context \\"AI sandbagging\\""}',
}, },
} }
], ],
@ -131,7 +131,7 @@ def test_unmatched_or_failed_result_does_not_prove_database_call() -> None:
"role": "tool", "role": "tool",
"tool_call_id": "call-missing", "tool_call_id": "call-missing",
"content": "Traceback (most recent call last): connection failed", "content": "Traceback (most recent call last): connection failed",
} },
] ]
assert extract_kb_tool_trace(unmatched)["database_tool_call_proven"] is False assert extract_kb_tool_trace(unmatched)["database_tool_call_proven"] is False

4
tests/test_run_leo_direct_claim_handler_suite.py
View file

@ -99,9 +99,7 @@ def test_safety_gate_passes_only_for_complete_no_send_no_write_run() -> None:
def test_safety_gate_rejects_write_capable_tool_and_incomplete_receipt() -> None: def test_safety_gate_rejects_write_capable_tool_and_incomplete_receipt() -> None:
report = safe_report() report = safe_report()
report["results"][0]["database_tool_trace"]["database_tool_calls_read_only"] = False report["results"][0]["database_tool_trace"]["database_tool_calls_read_only"] = False
report["results"][0]["database_tool_trace"]["calls"][0]["database_invocations"][0][ report["results"][0]["database_tool_trace"]["calls"][0]["database_invocations"][0]["access_mode"] = "write"
"access_mode"
] = "write"
report["execution_manifest_summary"]["all_turns_attribution_complete"] = False report["execution_manifest_summary"]["all_turns_attribution_complete"] = False
gate = suite.build_safety_gate(report) gate = suite.build_safety_gate(report)

40
tests/test_run_leo_m3taversal_oos_handler_suite.py
View file

@ -58,7 +58,7 @@ def test_guarded_remote_script_compiles_and_installs_preexecution_gate(grounding
assert "timeout=180" in script assert "timeout=180" in script
assert "timeout=300" not in script assert "timeout=300" not in script
if grounding_mode == "db_tool_ablated": if grounding_mode == "db_tool_ablated":
assert 'shutil.rmtree(db_context_dir)' in script assert "shutil.rmtree(db_context_dir)" in script
def test_prompt_leakage_scan_uses_partial_markers_and_runtime_roots( def test_prompt_leakage_scan_uses_partial_markers_and_runtime_roots(
@ -82,6 +82,44 @@ def test_current_repo_runtime_has_no_frozen_prompt_marker_leakage() -> None:
assert scan["scanned_files"] > 0 assert scan["scanned_files"] > 0
@pytest.mark.parametrize("root_state", ("missing", "empty", "symlink"))
def test_prompt_leakage_scan_fails_closed_without_real_coverage(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, root_state: str
) -> None:
runtime_root = tmp_path / "runtime"
if root_state == "empty":
runtime_root.mkdir()
elif root_state == "symlink":
target = tmp_path / "target"
target.mkdir()
(target / "runtime.py").write_text("safe runtime text", encoding="utf-8")
runtime_root.symlink_to(target, target_is_directory=True)
monkeypatch.setattr(suite, "RUNTIME_PROMPT_SCAN_ROOTS", (runtime_root,))
scan = suite.prompt_leakage_scan(protocol())
assert scan["pass"] is False
assert scan["expected_roots"][0]["exists"] is (root_state != "missing")
assert scan["scanned_files"] == 0
def test_prompt_leakage_scan_fails_closed_on_nested_symlink(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
runtime_root = tmp_path / "runtime"
runtime_root.mkdir()
(runtime_root / "runtime.py").write_text("safe runtime text", encoding="utf-8")
hidden = tmp_path / "hidden"
hidden.mkdir()
(hidden / "prompt.md").write_text("unscanned prompt text", encoding="utf-8")
(runtime_root / "linked").symlink_to(hidden, target_is_directory=True)
monkeypatch.setattr(suite, "RUNTIME_PROMPT_SCAN_ROOTS", (runtime_root,))
scan = suite.prompt_leakage_scan(protocol())
assert scan["pass"] is False
assert scan["scanned_files"] == 1
assert len(scan["symlink_entries"]) == 1
def test_restart_probe_requires_full_fingerprint_guard_and_cleanup() -> None: def test_restart_probe_requires_full_fingerprint_guard_and_cleanup() -> None:
counts = {"public.claims": 2} counts = {"public.claims": 2}
fingerprint = {"status": "ok", "fingerprint_sha256": "a" * 64} fingerprint = {"status": "ok", "fingerprint_sha256": "a" * 64}

8
tests/test_working_leo_m3taversal_oos_benchmark.py
View file

@ -166,9 +166,7 @@ def test_oos_score_passes_complete_behavior_and_memory_pair() -> None:
assert score["pass"] is True assert score["pass"] is True
assert score["passes"] == 16 assert score["passes"] == 16
assert score["memory_continuity"]["same_blocker_recalled"] is True assert score["memory_continuity"]["same_blocker_recalled"] is True
assert {"approved", "applied", "canonical"} <= set( assert {"approved", "applied", "canonical"} <= set(score["memory_continuity"]["recall_overlap_terms"])
score["memory_continuity"]["recall_overlap_terms"]
)
def test_oos_autonomous_retrieval_semantics_reject_generic_challenge() -> None: def test_oos_autonomous_retrieval_semantics_reject_generic_challenge() -> None:
@ -559,9 +557,7 @@ def test_oos_composition_rejects_belief_update_in_supported_apply_list() -> None
"(claims, sources, evidence, edges, reasoning_tools, belief updates). " "(claims, sources, evidence, edges, reasoning_tools, belief updates). "
"behavioral_rules and governance_gates remain staged for a separate reviewed apply capability." "behavioral_rules and governance_gates remain staged for a separate reviewed apply capability."
) )
assert benchmark.composition_capability_issues("OOS-06", reply) == [ assert benchmark.composition_capability_issues("OOS-06", reply) == ["unsupported_collection_listed_as_applyable"]
"unsupported_collection_listed_as_applyable"
]
def test_db_compiled_responses_pass_composition_and_source_intake_benchmarks() -> None: def test_db_compiled_responses_pass_composition_and_source_intake_benchmarks() -> None:

131
tests/test_working_leo_m3taversal_oos_protocol.py
View file

@ -83,9 +83,7 @@ def retrieval_receipt(query_sha256: str) -> dict:
def rehash_retrieval_receipt(receipt: dict) -> None: def rehash_retrieval_receipt(receipt: dict) -> None:
trace_payload = { trace_payload = {
key: value key: value for key, value in receipt.items() if key not in {"receipt_sha256", "trace_payload_sha256"}
for key, value in receipt.items()
if key not in {"receipt_sha256", "trace_payload_sha256"}
} }
receipt["trace_payload_sha256"] = hashlib.sha256( receipt["trace_payload_sha256"] = hashlib.sha256(
json.dumps(trace_payload, sort_keys=True, separators=(",", ":")).encode("utf-8") json.dumps(trace_payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
@ -289,9 +287,7 @@ def attach_fake_execution_manifests(
} }
previous_execution_sha256 = None previous_execution_sha256 = None
allowed_missing = ( allowed_missing = (
protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING if grounded else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING
if grounded
else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING
) )
fingerprint = report["db_fingerprint_before"] fingerprint = report["db_fingerprint_before"]
counts_sha256 = protocol_lib.canonical_sha256({}) counts_sha256 = protocol_lib.canonical_sha256({})
@ -319,9 +315,7 @@ def attach_fake_execution_manifests(
"runtime": { "runtime": {
"behavior_sha256": report["executed_behavior_manifest"]["behavior_sha256"], "behavior_sha256": report["executed_behavior_manifest"]["behavior_sha256"],
"hermes_runtime": report["executed_behavior_manifest"]["hermes_runtime"], "hermes_runtime": report["executed_behavior_manifest"]["hermes_runtime"],
"teleo_infrastructure_runtime": report["executed_behavior_manifest"][ "teleo_infrastructure_runtime": report["executed_behavior_manifest"]["teleo_infrastructure_runtime"],
"teleo_infrastructure_runtime"
],
"harness_source": harness_source, "harness_source": harness_source,
}, },
"model_execution": { "model_execution": {
@ -475,8 +469,7 @@ def fake_report(protocol: dict, trial: dict, *, grounded: bool) -> dict:
"remote_temp_profile_prompt_leakage_scan": { "remote_temp_profile_prompt_leakage_scan": {
"scope": "full_model_visible_temp_profile_excluding_sessions_state_memories_and_venv", "scope": "full_model_visible_temp_profile_excluding_sessions_state_memories_and_venv",
"expected_roots": [ "expected_roots": [
{"name": name, "exists": True, "is_symlink": False} {"name": name, "exists": True, "is_symlink": False} for name in ("profile", "skills", "plugins", "bin")
for name in ("profile", "skills", "plugins", "bin")
], ],
"scanned_files": 10, "scanned_files": 10,
"scanned_bytes": 1000, "scanned_bytes": 1000,
@ -520,9 +513,7 @@ def restart_receipt(protocol: dict, directory: Path, trial: dict) -> dict:
def probe(path: Path, *, before_restart: bool) -> dict: def probe(path: Path, *, before_restart: bool) -> dict:
payload = { payload = {
"generated_at_utc": "2026-07-15T00:00:00+00:00" "generated_at_utc": "2026-07-15T00:00:00+00:00" if before_restart else "2026-07-15T00:01:15+00:00",
if before_restart
else "2026-07-15T00:01:15+00:00",
"protocol_id": protocol["protocol_id"], "protocol_id": protocol["protocol_id"],
"protocol_hash_sha256": protocol["protocol_hash_sha256"], "protocol_hash_sha256": protocol["protocol_hash_sha256"],
"source_hashes": protocol["source_hashes"], "source_hashes": protocol["source_hashes"],
@ -633,19 +624,14 @@ def test_protocol_freezes_three_unique_variants_and_follow_up_shapes() -> None:
assert protocol["trials"][-1]["session_mode"] == "post_restart_clean_session" assert protocol["trials"][-1]["session_mode"] == "post_restart_clean_session"
for family_id in protocol["blinding"]["prompt_families"]: for family_id in protocol["blinding"]["prompt_families"]:
prompts = [ prompts = [
next(item for item in trial["prompts"] if item["family_id"] == family_id) next(item for item in trial["prompts"] if item["family_id"] == family_id) for trial in protocol["trials"]
for trial in protocol["trials"]
] ]
assert len({item["variant_index"] for item in prompts}) == 3 assert len({item["variant_index"] for item in prompts}) == 3
assert len({item["expected_follow_up"] for item in prompts}) == 3 assert len({item["expected_follow_up"] for item in prompts}) == 3
assert all(item["leakage_markers"] for item in prompts) assert all(item["leakage_markers"] for item in prompts)
assert all("row id:" not in item["message"].lower() for item in prompts) assert all("row id:" not in item["message"].lower() for item in prompts)
retrieval_prompts = [ retrieval_prompts = [
next( next(item for item in trial["prompts"] if item["family_id"] == "autonomous_retrieval_reasoning")
item
for item in trial["prompts"]
if item["family_id"] == "autonomous_retrieval_reasoning"
)
for trial in protocol["trials"] for trial in protocol["trials"]
] ]
assert len({item["subject"] for item in retrieval_prompts}) == 3 assert len({item["subject"] for item in retrieval_prompts}) == 3
@ -669,9 +655,10 @@ def test_protocol_validation_rejects_prompt_and_source_hash_tampering() -> None:
protocol["protocol_hash_sha256"] = protocol_lib.canonical_sha256( protocol["protocol_hash_sha256"] = protocol_lib.canonical_sha256(
{key: value for key, value in protocol.items() if key != "protocol_hash_sha256"} {key: value for key, value in protocol.items() if key != "protocol_hash_sha256"}
) )
assert "source_changed_after_freeze:readonly_guard_sha256" in protocol_lib.validate_protocol( assert (
protocol, verify_source_hashes=True "source_changed_after_freeze:readonly_guard_sha256"
)["issues"] in protocol_lib.validate_protocol(protocol, verify_source_hashes=True)["issues"]
)
def test_live_trial_requires_semantics_subject_receipts_and_real_ablation() -> None: def test_live_trial_requires_semantics_subject_receipts_and_real_ablation() -> None:
@ -718,11 +705,7 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
manifest = report["results"][0]["execution_manifest"] manifest = report["results"][0]["execution_manifest"]
manifest["attribution"]["missing_required_bindings"].append("unexpected_gap") manifest["attribution"]["missing_required_bindings"].append("unexpected_gap")
stable = { stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
key: value
for key, value in manifest.items()
if key not in {"generated_at_utc", "execution_sha256"}
}
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable) manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
validation = protocol_lib._benchmark_execution_chain( validation = protocol_lib._benchmark_execution_chain(
report, report,
@ -734,11 +717,7 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
ablated = fake_report(protocol, trial, grounded=False) ablated = fake_report(protocol, trial, grounded=False)
manifest = ablated["results"][-1]["execution_manifest"] manifest = ablated["results"][-1]["execution_manifest"]
manifest["attribution"]["missing_required_bindings"].append("database_tool_results") manifest["attribution"]["missing_required_bindings"].append("database_tool_results")
stable = { stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
key: value
for key, value in manifest.items()
if key not in {"generated_at_utc", "execution_sha256"}
}
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable) manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
validation = protocol_lib._benchmark_execution_chain( validation = protocol_lib._benchmark_execution_chain(
ablated, ablated,
@ -786,22 +765,14 @@ def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal
for result in baseline["results"]: for result in baseline["results"]:
manifest = result["execution_manifest"] manifest = result["execution_manifest"]
manifest["runtime"]["behavior_sha256"] = behavior["behavior_sha256"] manifest["runtime"]["behavior_sha256"] = behavior["behavior_sha256"]
stable = { stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
key: value
for key, value in manifest.items()
if key not in {"generated_at_utc", "execution_sha256"}
}
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable) manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
for previous, result in zip(baseline["results"], baseline["results"][1:], strict=False): for previous, result in zip(baseline["results"], baseline["results"][1:], strict=False):
manifest = result["execution_manifest"] manifest = result["execution_manifest"]
manifest["session_boundary"]["conversation"]["previous_execution_sha256"] = previous[ manifest["session_boundary"]["conversation"]["previous_execution_sha256"] = previous["execution_manifest"][
"execution_manifest" "execution_sha256"
]["execution_sha256"] ]
stable = { stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
key: value
for key, value in manifest.items()
if key not in {"generated_at_utc", "execution_sha256"}
}
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable) manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
score = protocol_lib.score_live_trial( score = protocol_lib.score_live_trial(
@ -812,10 +783,7 @@ def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal
) )
assert score["pass"] is False assert score["pass"] is False
assert score["executed_behavior_ablation"]["checks"]["non_middleware_components_equal"] is False assert score["executed_behavior_ablation"]["checks"]["non_middleware_components_equal"] is False
assert ( assert score["comparison_axis_checks"]["executed_behavior_manifests_capture_only_declared_ablation"] is False
score["comparison_axis_checks"]["executed_behavior_manifests_capture_only_declared_ablation"]
is False
)
def test_relative_retained_paths_resolve_from_repo_root() -> None: def test_relative_retained_paths_resolve_from_repo_root() -> None:
@ -848,9 +816,7 @@ def test_live_trial_rejects_duplicate_results_malformed_receipts_and_invented_id
first = score["prompt_scores"][0] first = score["prompt_scores"][0]
assert score["pass"] is False assert score["pass"] is False
assert first["receipt_pass"] is False assert first["receipt_pass"] is False
assert first["receipt_score"]["unsupported_identifiers"] == [ assert first["receipt_score"]["unsupported_identifiers"] == ["12345678-1234-4123-8123-123456789abc"]
"12345678-1234-4123-8123-123456789abc"
]
grounded = fake_report(protocol, trial, grounded=True) grounded = fake_report(protocol, trial, grounded=True)
traced_receipt = grounded["results"][0]["database_context_trace"][0]["retrieval_receipt"] traced_receipt = grounded["results"][0]["database_context_trace"][0]["retrieval_receipt"]
@ -895,11 +861,7 @@ def test_live_trial_rejects_duplicate_results_malformed_receipts_and_invented_id
def test_autonomous_retrieval_requires_nonempty_rows_and_receipted_reply_ids() -> None: def test_autonomous_retrieval_requires_nonempty_rows_and_receipted_reply_ids() -> None:
protocol = frozen_protocol() protocol = frozen_protocol()
trial = protocol["trials"][0] trial = protocol["trials"][0]
prompt = next( prompt = next(item for item in trial["prompts"] if item["family_id"] == "autonomous_retrieval_reasoning")
item
for item in trial["prompts"]
if item["family_id"] == "autonomous_retrieval_reasoning"
)
grounded = fake_report(protocol, trial, grounded=True) grounded = fake_report(protocol, trial, grounded=True)
result = next(item for item in grounded["results"] if item["prompt_id"] == prompt["id"]) result = next(item for item in grounded["results"] if item["prompt_id"] == prompt["id"])
@ -993,11 +955,7 @@ def test_autonomous_retrieval_requires_nonempty_rows_and_receipted_reply_ids() -
def test_autonomous_retrieval_ablation_rejects_generic_prose_and_invented_ids() -> None: def test_autonomous_retrieval_ablation_rejects_generic_prose_and_invented_ids() -> None:
protocol = frozen_protocol() protocol = frozen_protocol()
trial = protocol["trials"][0] trial = protocol["trials"][0]
prompt = next( prompt = next(item for item in trial["prompts"] if item["family_id"] == "autonomous_retrieval_reasoning")
item
for item in trial["prompts"]
if item["family_id"] == "autonomous_retrieval_reasoning"
)
grounded = fake_report(protocol, trial, grounded=True) grounded = fake_report(protocol, trial, grounded=True)
ablated = fake_report(protocol, trial, grounded=False) ablated = fake_report(protocol, trial, grounded=False)
result = next(item for item in ablated["results"] if item["prompt_id"] == prompt["id"]) result = next(item for item in ablated["results"] if item["prompt_id"] == prompt["id"])
@ -1010,9 +968,7 @@ def test_autonomous_retrieval_ablation_rejects_generic_prose_and_invented_ids()
"Challenge: the evidence does not prove the broader leap. " "Challenge: the evidence does not prove the broader leap. "
"Narrower revision: only the generic source is supported; no mutation was made." "Narrower revision: only the generic source is supported; no mutation was made."
) )
result["execution_manifest"]["turn"]["reply_sha256"] = hashlib.sha256( result["execution_manifest"]["turn"]["reply_sha256"] = hashlib.sha256(result["reply"].encode()).hexdigest()
result["reply"].encode()
).hexdigest()
result["execution_manifest"]["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256( result["execution_manifest"]["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(
{ {
key: value key: value
@ -1064,9 +1020,7 @@ def test_hash_bound_contract_row_ids_are_supported_but_unreceipted_ids_fail() ->
def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words() -> None: def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words() -> None:
protocol = frozen_protocol() protocol = frozen_protocol()
source_prompt = next( source_prompt = next(item for item in protocol["trials"][0]["prompts"] if item["family_id"] == "source_evidence")
item for item in protocol["trials"][0]["prompts"] if item["family_id"] == "source_evidence"
)
wrong_subject = "The database proposal is approved but not applied, so wait for a canonical receipt." wrong_subject = "The database proposal is approved but not applied, so wait for a canonical receipt."
assert protocol_lib._subject_alignment(source_prompt, wrong_subject) is False assert protocol_lib._subject_alignment(source_prompt, wrong_subject) is False
generic_family_only = "The source document has claim_evidence and evidence, but the named packet is omitted." generic_family_only = "The source document has claim_evidence and evidence, but the named packet is omitted."
@ -1077,15 +1031,11 @@ def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words
) )
assert protocol_lib._subject_alignment(source_prompt, duplicate_subject) is False assert protocol_lib._subject_alignment(source_prompt, duplicate_subject) is False
sibling_subject = next(item for item in source_prompt["family_subjects"] if item != source_prompt["subject"]) sibling_subject = next(item for item in source_prompt["family_subjects"] if item != source_prompt["subject"])
shotgun = ( shotgun = f"{source_prompt['subject']} and {sibling_subject} both use source evidence and claim_evidence."
f"{source_prompt['subject']} and {sibling_subject} both use source evidence and claim_evidence."
)
assert protocol_lib._subject_alignment(source_prompt, shotgun) is False assert protocol_lib._subject_alignment(source_prompt, shotgun) is False
retrieval_prompt = next( retrieval_prompt = next(
item item for item in protocol["trials"][0]["prompts"] if item["family_id"] == "autonomous_retrieval_reasoning"
for item in protocol["trials"][0]["prompts"]
if item["family_id"] == "autonomous_retrieval_reasoning"
) )
good = ( good = (
f"{retrieval_prompt['subject']}. Claim body and source evidence are distinct. " f"{retrieval_prompt['subject']}. Claim body and source evidence are distinct. "
@ -1230,15 +1180,24 @@ def test_receipt_discriminator_rejects_wrong_command_extra_call_and_wrong_token(
def test_aggregate_recomputes_integrity_and_rejects_forged_minimal_scores(tmp_path: Path) -> None: def test_aggregate_recomputes_integrity_and_rejects_forged_minimal_scores(tmp_path: Path) -> None:
protocol = frozen_protocol() protocol = frozen_protocol()
scores = [ scores = [score_for_trial(protocol, trial, tmp_path / trial["trial_id"]) for trial in protocol["trials"]]
score_for_trial(protocol, trial, tmp_path / trial["trial_id"])
for trial in protocol["trials"]
]
aggregate = protocol_lib.aggregate_trial_scores(protocol, scores) aggregate = protocol_lib.aggregate_trial_scores(protocol, scores)
assert aggregate["pass"] is True assert aggregate["evaluation_pass"] is True
assert aggregate["machine_verifiable_t3_pass"] is False
assert aggregate["pass"] is False
assert aggregate["current_tier"] == "T2_runtime_artifact_validation"
assert aggregate["independent_live_attestation"]["pass"] is False
assert aggregate["mean_current_grounded_pass_rate"] == 1.0 assert aggregate["mean_current_grounded_pass_rate"] == 1.0
assert aggregate["mean_ablation_grounded_pass_rate"] == 0.0 assert aggregate["mean_ablation_grounded_pass_rate"] == 0.0
caller_attested = copy.deepcopy(scores)
for score in caller_attested:
score["independent_live_attestation"] = {"pass": True, "issuer": "caller"}
rejected_attestation = protocol_lib.aggregate_trial_scores(protocol, caller_attested)
assert rejected_attestation["evaluation_pass"] is False
assert rejected_attestation["machine_verifiable_t3_pass"] is False
assert rejected_attestation["pass"] is False
forged = [ forged = [
{ {
"trial_id": trial["trial_id"], "trial_id": trial["trial_id"],
@ -1258,8 +1217,7 @@ def test_aggregate_recomputes_integrity_and_rejects_forged_minimal_scores(tmp_pa
rejected = protocol_lib.aggregate_trial_scores(protocol, tampered) rejected = protocol_lib.aggregate_trial_scores(protocol, tampered)
assert rejected["pass"] is False assert rejected["pass"] is False
assert ( assert (
rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]]["checks"] rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]]["checks"]["comparison_axis_checks_passed"]
["comparison_axis_checks_passed"]
is False is False
) )
@ -1269,9 +1227,7 @@ def test_aggregate_recomputes_integrity_and_rejects_forged_minimal_scores(tmp_pa
failing_report["results"][0]["reply"] = "This answer is unrelated and ungrounded." failing_report["results"][0]["reply"] = "This answer is unrelated and ungrounded."
report_path.write_text(json.dumps(failing_report, sort_keys=True) + "\n", encoding="utf-8") report_path.write_text(json.dumps(failing_report, sort_keys=True) + "\n", encoding="utf-8")
forged_from_failing_report[0]["grounded_report_sha256"] = hashlib.sha256(report_path.read_bytes()).hexdigest() forged_from_failing_report[0]["grounded_report_sha256"] = hashlib.sha256(report_path.read_bytes()).hexdigest()
forged_from_failing_report[0]["grounded_report_payload_sha256"] = protocol_lib.canonical_sha256( forged_from_failing_report[0]["grounded_report_payload_sha256"] = protocol_lib.canonical_sha256(failing_report)
failing_report
)
forged_from_failing_report[0]["derivation_core_sha256"] = protocol_lib.canonical_sha256( forged_from_failing_report[0]["derivation_core_sha256"] = protocol_lib.canonical_sha256(
protocol_lib.score_derivation_core(forged_from_failing_report[0]) protocol_lib.score_derivation_core(forged_from_failing_report[0])
) )
@ -1285,7 +1241,6 @@ def test_aggregate_recomputes_integrity_and_rejects_forged_minimal_scores(tmp_pa
rejected = protocol_lib.aggregate_trial_scores(protocol, forged_from_failing_report) rejected = protocol_lib.aggregate_trial_scores(protocol, forged_from_failing_report)
assert rejected["pass"] is False assert rejected["pass"] is False
assert ( assert (
rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]]["checks"] rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]]["checks"]["grounded_report_artifact_bound"]
["grounded_report_artifact_bound"]
is False is False
) )