Merge pull request #164 from living-ip/codex/leo-oos-reasoning-benchmark-review-fix-20260715
Harden OOS benchmark proof boundaries
This commit is contained in:
twentyOne2x
GitHub
commit
7bdca5d992
16 changed files with 319 additions and 1625 deletions
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"schema": "livingip.leoEvaluatedOosProtocolManifest.v1",
|
||||
"protocol_schema": "livingip.leoM3taversalOosProtocol.v2",
|
||||
"protocol_id": "leo-m3taversal-oos-8c0fc263b5cd95b8",
|
||||
"protocol_hash_sha256": "d68330b20d31677b231beb8450d5fdfd40efebf498cfc2f8ae2e63efd860bf43",
|
||||
"protocol_file_sha256": "fac462ee83b1035215cca12262e27b70c70e8a8ed0a216aa70c69ef46cd6a185",
|
||||
"harness_git_head": "93eddbcd4f27bd9e349f217767da2c9bf2b7baeb",
|
||||
"trial_count": 3,
|
||||
"prompt_count": 30,
|
||||
"family_count": 10,
|
||||
"unique_prompt_id_count": 30,
|
||||
"unique_message_hash_count": 30,
|
||||
"validation_pass": true,
|
||||
"prompt_bodies_committed": false,
|
||||
"retention_contract": "The exact post-evaluation protocol is retained as a private mode-0600 artifact and is bound by protocol_file_sha256. This manifest is not sufficient to recreate prompt bodies by itself.",
|
||||
"claim_ceiling": "This manifest identifies the exact evaluated protocol without publishing the formerly blinded prompts. It does not independently authenticate live VPS execution."
|
||||
}
|
||||
|
|
@ -203,6 +203,7 @@ STOPWORDS = {
|
|||
"with",
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False)
|
||||
parser.add_argument("--ssh", default="teleo@77.42.65.182")
|
||||
|
|
@ -461,8 +462,10 @@ def _has_unnegated_database_state_request(query: str) -> bool:
|
|||
segment,
|
||||
)
|
||||
scope_start = scope_match.start() if scope_match else -1
|
||||
if scope_start < 0 and "canonical" in segment and any(
|
||||
term in segment for term in ("approved", "applied", "proposal")
|
||||
if (
|
||||
scope_start < 0
|
||||
and "canonical" in segment
|
||||
and any(term in segment for term in ("approved", "applied", "proposal"))
|
||||
):
|
||||
scope_start = segment.index("canonical")
|
||||
negation_before_scope = any(
|
||||
|
|
@ -522,11 +525,7 @@ def rank_query_matching_proposals(
|
|||
ranked: list[tuple[int, int, str, int, dict[str, Any], list[str]]] = []
|
||||
for index, proposal in enumerate(proposals):
|
||||
haystack = json.dumps(proposal, sort_keys=True, default=str).lower()
|
||||
matched_terms = [
|
||||
term
|
||||
for term in terms
|
||||
if re.search(rf"(?<![a-z0-9]){re.escape(term)}(?![a-z0-9])", haystack)
|
||||
]
|
||||
matched_terms = [term for term in terms if re.search(rf"(?<![a-z0-9]){re.escape(term)}(?![a-z0-9])", haystack)]
|
||||
if len(matched_terms) < minimum_term_matches or not version_terms.issubset(matched_terms):
|
||||
continue
|
||||
score = sum(2 if len(term) >= 8 or re.fullmatch(r"v\d+", term) else 1 for term in matched_terms)
|
||||
|
|
@ -591,8 +590,7 @@ def operational_contracts(
|
|||
)
|
||||
)
|
||||
and any(
|
||||
term in lowered
|
||||
for term in ("wrong", "missing", "stuck", "link", "point", "canonical evidence", "audit")
|
||||
term in lowered for term in ("wrong", "missing", "stuck", "link", "point", "canonical evidence", "audit")
|
||||
)
|
||||
)
|
||||
kb_scope = (
|
||||
|
|
@ -635,9 +633,7 @@ def operational_contracts(
|
|||
)
|
||||
)
|
||||
)
|
||||
broad_kb_change_question = _has_unnegated_action(
|
||||
query, ("change", "changed", "update", "updated", "landed")
|
||||
)
|
||||
broad_kb_change_question = _has_unnegated_action(query, ("change", "changed", "update", "updated", "landed"))
|
||||
proposal_state_question = (
|
||||
kb_scope
|
||||
and not forecast_resolution_question
|
||||
|
|
@ -936,9 +932,7 @@ def operational_contracts(
|
|||
contracts.append(
|
||||
{
|
||||
"id": "forecast_resolution_schema",
|
||||
"current_columns": {
|
||||
name: list(schema.get(name, ())) for name in ("claims", "claim_edges")
|
||||
},
|
||||
"current_columns": {name: list(schema.get(name, ())) for name in ("claims", "claim_edges")},
|
||||
"history_boundary": "preserve the original forecast and its missing resolution criteria",
|
||||
"schema_boundary": (
|
||||
"do not invent forecast-resolution fields or a resolves edge; stage a reviewed schema proposal"
|
||||
|
|
@ -955,9 +949,7 @@ def operational_contracts(
|
|||
contracts.append(
|
||||
{
|
||||
"id": "claim_supersession_schema",
|
||||
"current_columns": {
|
||||
name: list(schema.get(name, ())) for name in ("claims", "claim_edges")
|
||||
},
|
||||
"current_columns": {name: list(schema.get(name, ())) for name in ("claims", "claim_edges")},
|
||||
"insert_boundary": "approve_claim may insert a replacement claim plus a claim-to-claim edge",
|
||||
"update_boundary": (
|
||||
"existing claim status and superseded_by updates require a separate reviewed apply capability"
|
||||
|
|
@ -967,7 +959,9 @@ def operational_contracts(
|
|||
|
||||
telegram_delivery_question = (
|
||||
"telegram" in lowered
|
||||
and any(term in lowered for term in ("gatewayrunner", "temporary-profile", "temporary profile", "posted nothing"))
|
||||
and any(
|
||||
term in lowered for term in ("gatewayrunner", "temporary-profile", "temporary profile", "posted nothing")
|
||||
)
|
||||
and any(term in lowered for term in ("proven", "delivery", "visible"))
|
||||
)
|
||||
if telegram_delivery_question:
|
||||
|
|
@ -1307,7 +1301,9 @@ def compile_operational_response(contracts: list[dict[str, Any]]) -> str | None:
|
|||
)
|
||||
elif status == "approved":
|
||||
lead = "No."
|
||||
state_sentence = "The matching proposal is reviewer-approved, but applied_at is empty; approved is not applied."
|
||||
state_sentence = (
|
||||
"The matching proposal is reviewer-approved, but applied_at is empty; approved is not applied."
|
||||
)
|
||||
elif status == "pending_review":
|
||||
lead = "No."
|
||||
state_sentence = (
|
||||
|
|
@ -2427,11 +2423,13 @@ def prepare_source(args: argparse.Namespace) -> dict[str, Any]:
|
|||
|
||||
def propose_source(args: argparse.Namespace) -> dict[str, Any]:
|
||||
if not args.local:
|
||||
raise SystemExit(
|
||||
"propose-source is VPS-local only; run it through the deployed teleo-kb wrapper on the VPS"
|
||||
)
|
||||
raise SystemExit("propose-source is VPS-local only; run it through the deployed teleo-kb wrapper on the VPS")
|
||||
receipt_path = args.receipt.expanduser().resolve()
|
||||
input_paths = {args.artifact.expanduser().resolve(), args.text.expanduser().resolve(), args.manifest.expanduser().resolve()}
|
||||
input_paths = {
|
||||
args.artifact.expanduser().resolve(),
|
||||
args.text.expanduser().resolve(),
|
||||
args.manifest.expanduser().resolve(),
|
||||
}
|
||||
if receipt_path in input_paths:
|
||||
raise SystemExit("--receipt must not overwrite an input file")
|
||||
|
||||
|
|
|
|||
|
|
@ -858,11 +858,7 @@ def _post_llm_call(**kwargs: Any) -> dict[str, str] | None:
|
|||
else:
|
||||
delivered = response
|
||||
hard_max = _reply_hard_max(contracts)
|
||||
budget_compacted = bool(
|
||||
not fail_closed
|
||||
and hard_max is not None
|
||||
and _word_count(delivered) > hard_max
|
||||
)
|
||||
budget_compacted = bool(not fail_closed and hard_max is not None and _word_count(delivered) > hard_max)
|
||||
if budget_compacted:
|
||||
delivered = _compact_response_to_budget(delivered, hard_max)
|
||||
delivered_issues = response_contract_issues(delivered, contracts)
|
||||
|
|
|
|||
|
|
@ -33,9 +33,7 @@ MUTATING_SUBCOMMANDS = frozenset(
|
|||
)
|
||||
KNOWN_SUBCOMMANDS = READ_ONLY_SUBCOMMANDS | MUTATING_SUBCOMMANDS
|
||||
|
||||
UUID_RE = re.compile(
|
||||
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b"
|
||||
)
|
||||
UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b")
|
||||
SHA256_RE = re.compile(r"\b[0-9a-fA-F]{64}\b")
|
||||
TELEO_KB_COMMAND_RE = re.compile(
|
||||
r"(?:^|[\s;&|()])(?:[^\s;&|()]+/)?teleo-kb\s+(?P<subcommand>[a-z][a-z0-9-]*)\b",
|
||||
|
|
@ -217,11 +215,7 @@ def extract_kb_tool_trace(messages: list[dict[str, Any]] | None) -> dict[str, An
|
|||
record["result"] = _sanitize_result(message.get("content"))
|
||||
|
||||
completed = [
|
||||
call
|
||||
for call in calls
|
||||
if call["result"]
|
||||
and call["result"]["nonempty"]
|
||||
and not call["result"]["error_detected"]
|
||||
call for call in calls if call["result"] and call["result"]["nonempty"] and not call["result"]["error_detected"]
|
||||
]
|
||||
receipt_calls = [
|
||||
call
|
||||
|
|
@ -230,11 +224,7 @@ def extract_kb_tool_trace(messages: list[dict[str, Any]] | None) -> dict[str, An
|
|||
and call["result"]["retrieval_receipt"].get("semantic_context_sha256")
|
||||
and call["result"]["retrieval_receipt"].get("artifact_state_sha256")
|
||||
]
|
||||
access_modes = {
|
||||
invocation["access_mode"]
|
||||
for call in calls
|
||||
for invocation in call["database_invocations"]
|
||||
}
|
||||
access_modes = {invocation["access_mode"] for call in calls for invocation in call["database_invocations"]}
|
||||
return {
|
||||
"schema": "livingip.leoKbToolTrace.v1",
|
||||
"database_tool_call_count": len(calls),
|
||||
|
|
|
|||
|
|
@ -112,9 +112,7 @@ def _safe_usage(value: Any) -> dict[str, int | float | None]:
|
|||
|
||||
def _trace_events(result: dict[str, Any], event: str) -> list[dict[str, Any]]:
|
||||
return [
|
||||
item
|
||||
for item in result.get("model_call_trace") or []
|
||||
if isinstance(item, dict) and item.get("event") == event
|
||||
item for item in result.get("model_call_trace") or [] if isinstance(item, dict) and item.get("event") == event
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -224,9 +222,7 @@ def _context_receipts(result: dict[str, Any], *, expected_query_sha256: str) ->
|
|||
or record.get("query_sha256") != expected_query_sha256
|
||||
):
|
||||
continue
|
||||
receipt = _safe_retrieval_receipt(
|
||||
record.get("retrieval_receipt"), expected_query_sha256=expected_query_sha256
|
||||
)
|
||||
receipt = _safe_retrieval_receipt(record.get("retrieval_receipt"), expected_query_sha256=expected_query_sha256)
|
||||
if receipt is not None:
|
||||
receipts.append(receipt)
|
||||
return receipts
|
||||
|
|
@ -238,7 +234,7 @@ def _tool_receipts(result: dict[str, Any]) -> list[dict[str, Any]]:
|
|||
for call in trace.get("calls") or []:
|
||||
if not isinstance(call, dict):
|
||||
continue
|
||||
raw = ((call.get("result") or {}).get("retrieval_receipt") or {})
|
||||
raw = (call.get("result") or {}).get("retrieval_receipt") or {}
|
||||
if (
|
||||
not isinstance(raw, dict)
|
||||
or raw.get("schema") != RETRIEVAL_RECEIPT_SCHEMA
|
||||
|
|
@ -273,9 +269,7 @@ def _database_required(result: dict[str, Any], *, prompt: str) -> bool:
|
|||
return bool(trace.get("database_tool_call_count"))
|
||||
|
||||
|
||||
def _database_context_binding(
|
||||
result: dict[str, Any], *, prompt_sha256: str, reply_sha256: str
|
||||
) -> dict[str, Any]:
|
||||
def _database_context_binding(result: dict[str, Any], *, prompt_sha256: str, reply_sha256: str) -> dict[str, Any]:
|
||||
records = [item for item in result.get("database_context_trace") or [] if isinstance(item, dict)]
|
||||
pre_records = [item for item in records if item.get("event") == "pre_llm_call"]
|
||||
post_records = [item for item in records if item.get("event") == "post_llm_call"]
|
||||
|
|
@ -371,8 +365,7 @@ def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]:
|
|||
and all(
|
||||
call.get("database_invocations")
|
||||
and all(
|
||||
invocation.get("access_mode") == "read_only"
|
||||
and _is_sha256(invocation.get("command_sha256"))
|
||||
invocation.get("access_mode") == "read_only" and _is_sha256(invocation.get("command_sha256"))
|
||||
for invocation in call.get("database_invocations") or []
|
||||
)
|
||||
for call in calls
|
||||
|
|
@ -433,10 +426,7 @@ def _conversation_binding(
|
|||
before = result.get("conversation_before") if isinstance(result.get("conversation_before"), dict) else {}
|
||||
after = result.get("conversation_after") if isinstance(result.get("conversation_after"), dict) else {}
|
||||
expected = expected_previous_conversation or {}
|
||||
before_valid = bool(
|
||||
_non_negative_int(before.get("message_count"))
|
||||
and _is_sha256(before.get("messages_sha256"))
|
||||
)
|
||||
before_valid = bool(_non_negative_int(before.get("message_count")) and _is_sha256(before.get("messages_sha256")))
|
||||
after_valid = bool(
|
||||
_non_negative_int(after.get("message_count"))
|
||||
and after.get("message_count", 0) > before.get("message_count", -1)
|
||||
|
|
@ -514,12 +504,9 @@ def _model_execution_binding(
|
|||
and post.get("prompt_sha256") == prompt_sha256
|
||||
),
|
||||
"raw_response_bound": bool(
|
||||
_is_sha256(raw_response_sha256)
|
||||
and post.get("raw_assistant_response_sha256") == raw_response_sha256
|
||||
),
|
||||
"delivered_response_bound": bool(
|
||||
response_hashes_valid and responses[-1].get("content_sha256") == reply_sha256
|
||||
_is_sha256(raw_response_sha256) and post.get("raw_assistant_response_sha256") == raw_response_sha256
|
||||
),
|
||||
"delivered_response_bound": bool(response_hashes_valid and responses[-1].get("content_sha256") == reply_sha256),
|
||||
"response_trace_count_matches_api_calls": len(responses) == len(calls),
|
||||
"api_call_sequence_valid": call_sequence == list(range(1, len(calls) + 1)),
|
||||
"session_binding_valid": len(session_hashes) == 1 and all(_is_sha256(item) for item in session_hashes),
|
||||
|
|
@ -575,8 +562,7 @@ def _required_missing(
|
|||
),
|
||||
"harness_git_head": _is_git_sha(harness_source.get("git_head")),
|
||||
"harness_worktree_clean": bool(
|
||||
harness_source.get("worktree_clean") is True
|
||||
and _is_sha256(harness_source.get("status_sha256"))
|
||||
harness_source.get("worktree_clean") is True and _is_sha256(harness_source.get("status_sha256"))
|
||||
),
|
||||
"actual_model_call": bool(
|
||||
model_execution.get("calls")
|
||||
|
|
@ -801,11 +787,7 @@ def validate_turn_manifest(manifest: dict[str, Any]) -> list[str]:
|
|||
problems = []
|
||||
if manifest.get("schema") != SCHEMA:
|
||||
problems.append("schema_mismatch")
|
||||
stable = {
|
||||
key: value
|
||||
for key, value in manifest.items()
|
||||
if key not in {"generated_at_utc", "execution_sha256"}
|
||||
}
|
||||
stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
|
||||
if manifest.get("execution_sha256") != canonical_sha256(stable):
|
||||
problems.append("execution_sha256_mismatch")
|
||||
missing = (manifest.get("attribution") or {}).get("missing_required_bindings")
|
||||
|
|
|
|||
|
|
@ -55,13 +55,11 @@ ASSIGNMENT_SECRET_PATTERN = re.compile(
|
|||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
DB_CONTEXT_PLUGIN = (
|
||||
ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py"
|
||||
)
|
||||
DB_CONTEXT_PLUGIN = ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py"
|
||||
KB_TOOL = ROOT / "hermes-agent" / "leoclean-bin" / "kb_tool.py"
|
||||
BEHAVIOR_MANIFEST = ROOT / "scripts" / "leo_behavior_manifest.py"
|
||||
|
||||
RESPONSE_BINDING_HELPER_SOURCE = r'''
|
||||
RESPONSE_BINDING_HELPER_SOURCE = r"""
|
||||
def response_bindings_are_hash_bound(results, records):
|
||||
if not isinstance(results, list) or not isinstance(records, list):
|
||||
return False
|
||||
|
|
@ -94,7 +92,7 @@ def response_bindings_are_hash_bound(results, records):
|
|||
and valid_sha256(item.get("response_sha256"))
|
||||
for item in record_rows
|
||||
)
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
REMOTE_SCRIPT = r'''
|
||||
|
|
@ -1010,9 +1008,7 @@ def build_safety_gate(report: dict[str, Any]) -> dict[str, Any]:
|
|||
handler = report.get("handler") if isinstance(report.get("handler"), dict) else {}
|
||||
service = report.get("service_before_after") if isinstance(report.get("service_before_after"), dict) else {}
|
||||
execution_summary = (
|
||||
report.get("execution_manifest_summary")
|
||||
if isinstance(report.get("execution_manifest_summary"), dict)
|
||||
else {}
|
||||
report.get("execution_manifest_summary") if isinstance(report.get("execution_manifest_summary"), dict) else {}
|
||||
)
|
||||
checks = {
|
||||
"remote_command_succeeded": report.get("remote_returncode") == 0,
|
||||
|
|
@ -1020,8 +1016,7 @@ def build_safety_gate(report: dict[str, Any]) -> dict[str, Any]:
|
|||
"handler_authorized": handler.get("authorized") is True,
|
||||
"results_nonempty": bool(results),
|
||||
"all_turns_returned_replies": bool(results) and all(item.get("ok") is True for item in results),
|
||||
"all_turns_declared_no_mutation": bool(results)
|
||||
and all(item.get("mutates_kb") is False for item in results),
|
||||
"all_turns_declared_no_mutation": bool(results) and all(item.get("mutates_kb") is False for item in results),
|
||||
"all_tool_traces_read_only_and_bound": bool(results)
|
||||
and all(_tool_trace_is_safe(item.get("database_tool_trace")) for item in results),
|
||||
"no_telegram_post": report.get("posted_to_telegram") is False,
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ RESULTS_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-current.json"
|
|||
SCORE_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.json"
|
||||
SCORE_MARKDOWN = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.md"
|
||||
PROTOCOL_REPORT_DIR = ROOT / "docs" / "reports" / "leo-oos-reasoning-benchmark-20260715"
|
||||
DEFAULT_PROTOCOL_JSON = PROTOCOL_REPORT_DIR / "blinded-protocol.json"
|
||||
GROUNDING_MODES = ("grounded", "db_tool_ablated")
|
||||
RUNTIME_PROMPT_SCAN_ROOTS = (
|
||||
ROOT / "hermes-agent" / "leoclean-skills",
|
||||
|
|
@ -45,27 +44,76 @@ def prompt_leakage_scan(protocol: dict[str, Any]) -> dict[str, Any]:
|
|||
]
|
||||
matches: list[dict[str, str]] = []
|
||||
scanned_files = 0
|
||||
scanned_bytes = 0
|
||||
scan_errors: list[dict[str, str]] = []
|
||||
symlink_entries: list[dict[str, str]] = []
|
||||
expected_roots = [
|
||||
{
|
||||
"path": str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path),
|
||||
"exists": path.exists(),
|
||||
"is_symlink": path.is_symlink(),
|
||||
}
|
||||
for path in RUNTIME_PROMPT_SCAN_ROOTS
|
||||
]
|
||||
for root in RUNTIME_PROMPT_SCAN_ROOTS:
|
||||
if not root.exists():
|
||||
if not root.exists() or root.is_symlink():
|
||||
continue
|
||||
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
||||
try:
|
||||
entries = sorted(root.rglob("*"))
|
||||
except OSError as exc:
|
||||
scan_errors.append(
|
||||
{
|
||||
"path_sha256": hashlib.sha256(str(root).encode()).hexdigest(),
|
||||
"error_type": type(exc).__name__,
|
||||
}
|
||||
)
|
||||
continue
|
||||
for path in entries:
|
||||
if path.is_symlink():
|
||||
symlink_entries.append({"path_sha256": hashlib.sha256(str(path).encode()).hexdigest()})
|
||||
paths = [item for item in entries if item.is_file() and not item.is_symlink()]
|
||||
for path in paths:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
except OSError as exc:
|
||||
scan_errors.append(
|
||||
{
|
||||
"path_sha256": hashlib.sha256(str(path).encode()).hexdigest(),
|
||||
"error_type": type(exc).__name__,
|
||||
}
|
||||
)
|
||||
continue
|
||||
scanned_files += 1
|
||||
scanned_bytes += len(text.encode())
|
||||
normalized = " ".join(re.findall(r"[a-z0-9_]+", text.lower()))
|
||||
for prompt_id, marker in prompt_markers:
|
||||
if marker in normalized:
|
||||
display_path = str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path)
|
||||
matches.append(
|
||||
{"prompt_id": prompt_id, "marker_sha256": hashlib.sha256(marker.encode()).hexdigest(), "path": display_path}
|
||||
{
|
||||
"prompt_id": prompt_id,
|
||||
"marker_sha256": hashlib.sha256(marker.encode()).hexdigest(),
|
||||
"path": display_path,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"scanned_roots": [str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path) for path in RUNTIME_PROMPT_SCAN_ROOTS],
|
||||
"expected_roots": expected_roots,
|
||||
"scanned_roots": [item["path"] for item in expected_roots],
|
||||
"scanned_files": scanned_files,
|
||||
"scanned_bytes": scanned_bytes,
|
||||
"errors": scan_errors,
|
||||
"symlink_entries": symlink_entries,
|
||||
"exact_prompt_matches": matches,
|
||||
"pass": not matches,
|
||||
"pass": bool(
|
||||
all(item["exists"] and not item["is_symlink"] for item in expected_roots)
|
||||
and scanned_files > 0
|
||||
and scanned_bytes > 0
|
||||
and not scan_errors
|
||||
and not symlink_entries
|
||||
and not matches
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -101,14 +149,14 @@ def build_guarded_remote_script(
|
|||
1,
|
||||
)
|
||||
tool_trace_source = (ROOT / "scripts" / "leo_tool_trace.py").read_text(encoding="utf-8")
|
||||
remote_tool_trace_import = ''' sys.path.insert(0, str(DEPLOY_ROOT / "scripts"))
|
||||
remote_tool_trace_import = """ sys.path.insert(0, str(DEPLOY_ROOT / "scripts"))
|
||||
from leo_tool_trace import extract_kb_tool_trace
|
||||
'''
|
||||
embedded_tool_trace_import = ''' tool_trace_module = types.ModuleType("leo_tool_trace_harness")
|
||||
"""
|
||||
embedded_tool_trace_import = """ tool_trace_module = types.ModuleType("leo_tool_trace_harness")
|
||||
tool_trace_module.__file__ = "<embedded leo_tool_trace.py>"
|
||||
exec(compile(LEO_TOOL_TRACE_SOURCE, tool_trace_module.__file__, "exec"), tool_trace_module.__dict__)
|
||||
extract_kb_tool_trace = tool_trace_module.extract_kb_tool_trace
|
||||
'''
|
||||
"""
|
||||
if script.count(remote_tool_trace_import) != 1:
|
||||
raise RuntimeError("remote leo_tool_trace import marker changed")
|
||||
script = script.replace(remote_tool_trace_import, embedded_tool_trace_import, 1)
|
||||
|
|
@ -758,7 +806,7 @@ def collect_restart_receipt(args: argparse.Namespace) -> int:
|
|||
restart = _ssh_readback(
|
||||
"systemctl restart leoclean-gateway.service && "
|
||||
"for i in $(seq 1 60); do "
|
||||
"if [ \"$(systemctl is-active leoclean-gateway.service)\" = active ]; then break; fi; sleep 1; done; "
|
||||
'if [ "$(systemctl is-active leoclean-gateway.service)" = active ]; then break; fi; sleep 1; done; '
|
||||
"systemctl show leoclean-gateway.service "
|
||||
"-p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp",
|
||||
timeout=120,
|
||||
|
|
@ -803,8 +851,7 @@ def collect_restart_receipt(args: argparse.Namespace) -> int:
|
|||
"db_fingerprint_unchanged": bool(
|
||||
fingerprint_before.get("status") == "ok"
|
||||
and fingerprint_after.get("status") == "ok"
|
||||
and fingerprint_before.get("fingerprint_sha256")
|
||||
== fingerprint_after.get("fingerprint_sha256")
|
||||
and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256")
|
||||
),
|
||||
"posted_to_telegram": False,
|
||||
"before_probe": {
|
||||
|
|
@ -898,8 +945,7 @@ def run_or_score_protocol_trial(args: argparse.Namespace) -> int:
|
|||
)
|
||||
mode_checks = {
|
||||
"expected_grounding_mode": report.get("grounding_mode") == args.grounding_mode,
|
||||
"db_context_state": report.get("db_context_plugin_enabled")
|
||||
is (args.grounding_mode == "grounded"),
|
||||
"db_context_state": report.get("db_context_plugin_enabled") is (args.grounding_mode == "grounded"),
|
||||
"tool_surface_mode": (report.get("read_only_tool_surface") or {}).get("mode")
|
||||
== ("read_only_kb" if args.grounding_mode == "grounded" else "no_db_ablation"),
|
||||
"zero_context_in_ablation": args.grounding_mode != "db_tool_ablated"
|
||||
|
|
|
|||
|
|
@ -26,9 +26,7 @@ EXPECTED_SOURCE_IDS = frozenset(
|
|||
)
|
||||
EXPECTED_SUBCOMMANDS = frozenset({"search", "show", "evidence"})
|
||||
SHA_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
UUID_RE = re.compile(
|
||||
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b"
|
||||
)
|
||||
UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b")
|
||||
WORD_RE = re.compile(r"\b\w+(?:[-']\w+)*\b")
|
||||
|
||||
|
||||
|
|
@ -100,13 +98,10 @@ def verify_report(
|
|||
and completed_without_error
|
||||
),
|
||||
"database_tools_read_only": (
|
||||
trace.get("database_tool_calls_read_only") is True
|
||||
and trace.get("access_modes") == ["read_only"]
|
||||
trace.get("database_tool_calls_read_only") is True and trace.get("access_modes") == ["read_only"]
|
||||
),
|
||||
"retrieval_receipt_proven": trace.get("database_retrieval_receipt_proven") is True,
|
||||
"claim_and_sources_returned": (
|
||||
EXPECTED_CLAIM_ID in row_ids and row_ids >= EXPECTED_SOURCE_IDS
|
||||
),
|
||||
"claim_and_sources_returned": (EXPECTED_CLAIM_ID in row_ids and row_ids >= EXPECTED_SOURCE_IDS),
|
||||
"claim_support_challenged": _contains_all(
|
||||
reply,
|
||||
("no_source_pointer", "grounds", "verified artifact", "illustrates"),
|
||||
|
|
@ -132,8 +127,7 @@ def verify_report(
|
|||
and post_record.get("validated") is True
|
||||
),
|
||||
"canonical_counts_unchanged": (
|
||||
report.get("db_counts_changed") is False
|
||||
and report.get("db_counts_before") == report.get("db_counts_after")
|
||||
report.get("db_counts_changed") is False and report.get("db_counts_before") == report.get("db_counts_after")
|
||||
),
|
||||
"canonical_content_fingerprint_unchanged": (
|
||||
report.get("db_fingerprint_unchanged") is True
|
||||
|
|
@ -159,10 +153,7 @@ def verify_report(
|
|||
and service_after.get("NRestarts") == "0"
|
||||
),
|
||||
"temporary_profile_removed": report.get("temp_profile_removed") is True,
|
||||
"deploy_head_matches_stamp": (
|
||||
bool(SHA_RE.fullmatch(deploy_head))
|
||||
and deploy_head == deploy_stamp
|
||||
),
|
||||
"deploy_head_matches_stamp": (bool(SHA_RE.fullmatch(deploy_head)) and deploy_head == deploy_stamp),
|
||||
}
|
||||
outcomes = {
|
||||
"found_the_relevant_claim_without_a_supplied_id": (
|
||||
|
|
@ -178,8 +169,7 @@ def verify_report(
|
|||
checks["live_behavior_profile_unchanged"] and checks["temporary_profile_removed"]
|
||||
),
|
||||
"did_not_change_canonical_knowledge": (
|
||||
checks["canonical_counts_unchanged"]
|
||||
and checks["canonical_content_fingerprint_unchanged"]
|
||||
checks["canonical_counts_unchanged"] and checks["canonical_content_fingerprint_unchanged"]
|
||||
),
|
||||
}
|
||||
passed = all(checks.values()) and all(outcomes.values())
|
||||
|
|
|
|||
|
|
@ -390,19 +390,19 @@ def harness_git_head() -> str:
|
|||
|
||||
|
||||
def instrument_db_context_plugin_source(source: str) -> str:
|
||||
marker = ''' safe["receipt_sha256"] = hashlib.sha256(
|
||||
marker = """ safe["receipt_sha256"] = hashlib.sha256(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
return safe
|
||||
'''
|
||||
replacement = ''' safe["trace_payload_sha256"] = hashlib.sha256(
|
||||
"""
|
||||
replacement = """ safe["trace_payload_sha256"] = hashlib.sha256(
|
||||
json.dumps(safe, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
safe["receipt_sha256"] = hashlib.sha256(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
return safe
|
||||
'''
|
||||
"""
|
||||
if source.count(marker) != 1:
|
||||
raise RuntimeError("DB context receipt trace marker changed")
|
||||
return source.replace(marker, replacement)
|
||||
|
|
@ -429,7 +429,12 @@ def source_paths() -> dict[str, Path]:
|
|||
"execution_manifest_sha256": scripts / "leo_turn_execution_manifest.py",
|
||||
"behavior_manifest_sha256": scripts / "leo_behavior_manifest.py",
|
||||
"tool_trace_sha256": scripts / "leo_tool_trace.py",
|
||||
"db_context_plugin_sha256": root / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py",
|
||||
"db_context_plugin_sha256": root
|
||||
/ "hermes-agent"
|
||||
/ "leoclean-plugins"
|
||||
/ "vps"
|
||||
/ "leo-db-context"
|
||||
/ "__init__.py",
|
||||
"db_context_plugin_manifest_sha256": root
|
||||
/ "hermes-agent"
|
||||
/ "leoclean-plugins"
|
||||
|
|
@ -641,9 +646,7 @@ def validate_protocol(protocol: dict[str, Any], *, verify_source_hashes: bool) -
|
|||
if requires_grounded_answer != (prompt.get("family_id") in AUTONOMOUS_RETRIEVAL_FAMILIES):
|
||||
issues.append(f"grounded_retrieval_answer_requirement_mismatch:{prompt_id}")
|
||||
if requires_grounded_answer and (
|
||||
"teleo-kb context" in message
|
||||
or "--limit" in message
|
||||
or "--context-limit" in message
|
||||
"teleo-kb context" in message or "--limit" in message or "--context-limit" in message
|
||||
):
|
||||
issues.append(f"autonomous_retrieval_exact_command_leak:{prompt_id}")
|
||||
if requires_tool_evidence and ("teleo-kb context" not in message or "`Receipt:" not in message):
|
||||
|
|
@ -685,9 +688,7 @@ def _subject_alignment(prompt: dict[str, Any], reply: str) -> bool:
|
|||
if str(item) != str(prompt.get("subject") or "")
|
||||
}
|
||||
matches = {
|
||||
str(anchor).lower()
|
||||
for anchor in prompt.get("subject_anchors") or []
|
||||
if str(anchor).lower() in reply.lower()
|
||||
str(anchor).lower() for anchor in prompt.get("subject_anchors") or [] if str(anchor).lower() in reply.lower()
|
||||
}
|
||||
return (
|
||||
bool(normalized_subject)
|
||||
|
|
@ -833,9 +834,7 @@ def _score_semantic_results(results: list[dict[str, Any]], trial: dict[str, Any]
|
|||
}
|
||||
|
||||
|
||||
def _executed_behavior_ablation(
|
||||
grounded_report: dict[str, Any], baseline_report: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
def _executed_behavior_ablation(grounded_report: dict[str, Any], baseline_report: dict[str, Any]) -> dict[str, Any]:
|
||||
grounded = grounded_report.get("executed_behavior_manifest") or {}
|
||||
baseline = baseline_report.get("executed_behavior_manifest") or {}
|
||||
stable_keys = {
|
||||
|
|
@ -884,30 +883,23 @@ def _executed_behavior_ablation(
|
|||
and _valid_sha256(baseline.get("behavior_sha256"))
|
||||
and baseline.get("behavior_sha256") == canonical_sha256(stable(baseline)),
|
||||
"behavior_hashes_differ": grounded.get("behavior_sha256") != baseline.get("behavior_sha256"),
|
||||
"component_sets_equal": bool(grounded_components)
|
||||
and set(grounded_components) == set(baseline_components),
|
||||
"component_sets_equal": bool(grounded_components) and set(grounded_components) == set(baseline_components),
|
||||
"non_middleware_components_equal": all(
|
||||
grounded_components.get(name) == baseline_components.get(name)
|
||||
for name in set(grounded_components) | set(baseline_components)
|
||||
if name != expected_component
|
||||
),
|
||||
"top_level_runtime_equal": all(
|
||||
grounded.get(key) == baseline.get(key)
|
||||
for key in stable_keys - {"components"}
|
||||
),
|
||||
"middleware_metadata_equal": {
|
||||
key: value for key, value in grounded_middleware.items() if key != "content"
|
||||
}
|
||||
"top_level_runtime_equal": all(grounded.get(key) == baseline.get(key) for key in stable_keys - {"components"}),
|
||||
"middleware_metadata_equal": {key: value for key, value in grounded_middleware.items() if key != "content"}
|
||||
== {key: value for key, value in baseline_middleware.items() if key != "content"},
|
||||
"middleware_nonfile_state_equal": grounded_content.get("missing") == baseline_content.get("missing")
|
||||
and grounded_content.get("symlinks") == baseline_content.get("symlinks"),
|
||||
"common_middleware_files_equal": bool(common_paths)
|
||||
and all(grounded_files[path] == baseline_files[path] for path in common_paths),
|
||||
"only_db_context_plugin_removed": extra_grounded_paths == expected_removed_paths
|
||||
and not extra_baseline_paths,
|
||||
"grounded_db_context_source_is_exact_instrumented_source": grounded_files.get(
|
||||
expected_db_context_path, {}
|
||||
).get("sha256")
|
||||
"only_db_context_plugin_removed": extra_grounded_paths == expected_removed_paths and not extra_baseline_paths,
|
||||
"grounded_db_context_source_is_exact_instrumented_source": grounded_files.get(expected_db_context_path, {}).get(
|
||||
"sha256"
|
||||
)
|
||||
== instrumented_plugin_sha256,
|
||||
"grounded_db_context_manifest_is_exact_frozen_source": grounded_files.get(
|
||||
expected_db_context_manifest_path, {}
|
||||
|
|
@ -976,15 +968,11 @@ def _receipt_score(
|
|||
)
|
||||
counts = receipt.get("counts") if isinstance(receipt.get("counts"), dict) else {}
|
||||
receipt_counts_typed = all(
|
||||
isinstance(counts.get(key), int)
|
||||
and not isinstance(counts.get(key), bool)
|
||||
and counts[key] >= 0
|
||||
isinstance(counts.get(key), int) and not isinstance(counts.get(key), bool) and counts[key] >= 0
|
||||
for key in ("claims", "context_rows", "evidence_rows")
|
||||
)
|
||||
safe_receipt_payload = {
|
||||
key: value
|
||||
for key, value in receipt.items()
|
||||
if key not in {"receipt_sha256", "trace_payload_sha256"}
|
||||
key: value for key, value in receipt.items() if key not in {"receipt_sha256", "trace_payload_sha256"}
|
||||
}
|
||||
trace_payload_sha256 = hashlib.sha256(
|
||||
json.dumps(safe_receipt_payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
|
|
@ -1000,10 +988,7 @@ def _receipt_score(
|
|||
and typed_attempts
|
||||
and (
|
||||
(consistency.get("status") == "stable_wal_marker" and wal_before == wal_after)
|
||||
or (
|
||||
consistency.get("status") == "stable_content_across_wal_change_retry"
|
||||
and attempts >= 2
|
||||
)
|
||||
or (consistency.get("status") == "stable_content_across_wal_change_retry" and attempts >= 2)
|
||||
)
|
||||
)
|
||||
if (
|
||||
|
|
@ -1017,8 +1002,7 @@ def _receipt_score(
|
|||
and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("injected_rows_sha256") or ""))
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("receipt_sha256") or ""))
|
||||
and receipt.get("trace_payload_sha256") == trace_payload_sha256
|
||||
and consistency.get("status")
|
||||
in {"stable_wal_marker", "stable_content_across_wal_change_retry"}
|
||||
and consistency.get("status") in {"stable_wal_marker", "stable_content_across_wal_change_retry"}
|
||||
and typed_attempts
|
||||
and consistency.get("database")
|
||||
and consistency.get("database_user")
|
||||
|
|
@ -1054,10 +1038,7 @@ def _receipt_score(
|
|||
reply_claim_citations = {match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(reply)}
|
||||
reply_source_citations = {match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(reply)}
|
||||
unsupported_identifiers = sorted(reply_identifiers - supported_identifiers)
|
||||
receipt_counts = [
|
||||
(item.get("retrieval_receipt") or {}).get("counts") or {}
|
||||
for item in retrieval_records
|
||||
]
|
||||
receipt_counts = [(item.get("retrieval_receipt") or {}).get("counts") or {} for item in retrieval_records]
|
||||
reply_sha256 = hashlib.sha256(reply.encode()).hexdigest()
|
||||
checks = {
|
||||
"reply_present": result.get("ok") is True and bool(str(result.get("reply") or "").strip()),
|
||||
|
|
@ -1073,8 +1054,7 @@ def _receipt_score(
|
|||
"post_contract_satisfaction_reported": post_contract_satisfaction_reported,
|
||||
"contract_ids_are_typed_lists": contract_ids_are_lists,
|
||||
"context_response_query_hash_bound": pre_hashes == post_hashes == {prompt_sha256},
|
||||
"delivered_response_hash_bound": len(post) == 1
|
||||
and post[0].get("delivered_response_sha256") == reply_sha256,
|
||||
"delivered_response_hash_bound": len(post) == 1 and post[0].get("delivered_response_sha256") == reply_sha256,
|
||||
"database_contract_present": bool(contract_ids - NON_DB_CONTRACT_IDS) if require_database_contract else True,
|
||||
"database_retrieval_receipt_present": bool(retrieval_records) if require_database_receipt else True,
|
||||
"grounded_claim_rows_nonempty": any(
|
||||
|
|
@ -1094,7 +1074,10 @@ def _receipt_score(
|
|||
if require_grounded_rows
|
||||
else True,
|
||||
"model_call_receipt_present": bool(model_call_trace)
|
||||
and any(item.get("event") == "post_api_request" and item.get("model") and item.get("provider") for item in model_call_trace),
|
||||
and any(
|
||||
item.get("event") == "post_api_request" and item.get("model") and item.get("provider")
|
||||
for item in model_call_trace
|
||||
),
|
||||
"conversation_history_prefix_preserved": result.get("conversation_history_prefix_preserved") is True,
|
||||
"no_unsupported_exact_identifiers": not unsupported_identifiers,
|
||||
}
|
||||
|
|
@ -1174,7 +1157,9 @@ def _benchmark_execution_chain(
|
|||
suite_safety = delivery.get("suite_safety") or {}
|
||||
attribution = manifest.get("attribution") or {}
|
||||
missing = attribution.get("missing_required_bindings")
|
||||
missing_set = set(missing) if isinstance(missing, list) and all(isinstance(item, str) for item in missing) else set()
|
||||
missing_set = (
|
||||
set(missing) if isinstance(missing, list) and all(isinstance(item, str) for item in missing) else set()
|
||||
)
|
||||
hermes_runtime = runtime.get("hermes_runtime") or {}
|
||||
teleo_runtime = runtime.get("teleo_infrastructure_runtime") or {}
|
||||
calls = model.get("calls") if isinstance(model.get("calls"), list) else []
|
||||
|
|
@ -1184,8 +1169,7 @@ def _benchmark_execution_chain(
|
|||
else []
|
||||
)
|
||||
checks = {
|
||||
"generic_manifest_valid": bool(manifest)
|
||||
and not execution_manifest_lib.validate_turn_manifest(manifest),
|
||||
"generic_manifest_valid": bool(manifest) and not execution_manifest_lib.validate_turn_manifest(manifest),
|
||||
"prompt_bound": turn.get("prompt_id") == result.get("prompt_id")
|
||||
and turn.get("prompt_sha256") == hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest(),
|
||||
"reply_bound": turn.get("reply_sha256")
|
||||
|
|
@ -1304,9 +1288,7 @@ def _top_level_safety(
|
|||
and all(value is True for key, value in handler_checks.items() if key != "all_turn_manifests_complete")
|
||||
)
|
||||
checks = {
|
||||
"fresh_temporary_session": (report.get("temp_profile_seed") or {}).get(
|
||||
"same_session_continuity_starts_fresh"
|
||||
)
|
||||
"fresh_temporary_session": (report.get("temp_profile_seed") or {}).get("same_session_continuity_starts_fresh")
|
||||
is True
|
||||
and bool(result_rows)
|
||||
and (result_rows[0].get("conversation_before") or {}).get("message_count") == 0,
|
||||
|
|
@ -1328,8 +1310,7 @@ def _top_level_safety(
|
|||
"database_fingerprint_after_ok": after.get("status") == "ok",
|
||||
"database_fingerprint_unchanged": report.get("db_fingerprint_unchanged") is True,
|
||||
"database_fingerprint_hash_equal": bool(
|
||||
before.get("fingerprint_sha256")
|
||||
and before.get("fingerprint_sha256") == after.get("fingerprint_sha256")
|
||||
before.get("fingerprint_sha256") and before.get("fingerprint_sha256") == after.get("fingerprint_sha256")
|
||||
),
|
||||
"live_behavior_manifest_unchanged": report.get("live_behavior_manifest_unchanged") is True,
|
||||
"service_unchanged_during_trial": service.get("unchanged_from_preexisting_live_readback") is True,
|
||||
|
|
@ -1407,9 +1388,9 @@ def _prompt_binding(report: dict[str, Any], trial: dict[str, Any]) -> dict[str,
|
|||
for prompt_id, prompt in expected.items():
|
||||
result = actual.get(prompt_id) or {}
|
||||
checks[f"prompt_text:{prompt_id}"] = result.get("prompt") == prompt["message"]
|
||||
checks[f"prompt_hash:{prompt_id}"] = hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest() == prompt[
|
||||
"message_sha256"
|
||||
]
|
||||
checks[f"prompt_hash:{prompt_id}"] = (
|
||||
hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest() == prompt["message_sha256"]
|
||||
)
|
||||
return {"checks": checks, "pass": all(checks.values())}
|
||||
|
||||
|
||||
|
|
@ -1434,7 +1415,9 @@ def _nonempty_integer_mapping(value: Any) -> bool:
|
|||
return bool(
|
||||
isinstance(value, dict)
|
||||
and value
|
||||
and all(isinstance(key, str) and isinstance(item, int) and not isinstance(item, bool) for key, item in value.items())
|
||||
and all(
|
||||
isinstance(key, str) and isinstance(item, int) and not isinstance(item, bool) for key, item in value.items()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1570,7 +1553,9 @@ def validate_restart_receipt(receipt: dict[str, Any] | None, report: dict[str, A
|
|||
"restart_command_succeeded": receipt.get("restart_returncode") == 0,
|
||||
"service_active_after": after.get("ActiveState") == "active" and after.get("SubState") == "running",
|
||||
"service_pid_changed": bool(before.get("MainPID") and before.get("MainPID") != after.get("MainPID")),
|
||||
"trial_observed_restarted_pid": bool(after.get("MainPID") and report_before.get("MainPID") == after.get("MainPID")),
|
||||
"trial_observed_restarted_pid": bool(
|
||||
after.get("MainPID") and report_before.get("MainPID") == after.get("MainPID")
|
||||
),
|
||||
"service_start_identity_bound": bool(
|
||||
after.get("ExecMainStartTimestamp")
|
||||
and after.get("ExecMainStartTimestamp") == report_before.get("ExecMainStartTimestamp")
|
||||
|
|
@ -1676,7 +1661,12 @@ def score_live_trial(
|
|||
"semantic_score": semantic_item,
|
||||
"receipt_score": receipt_item,
|
||||
"reply_sha256": hashlib.sha256(
|
||||
str((next((row for row in report_results if row.get("prompt_id") == prompt_id), {}) or {}).get("reply") or "").encode()
|
||||
str(
|
||||
(next((row for row in report_results if row.get("prompt_id") == prompt_id), {}) or {}).get(
|
||||
"reply"
|
||||
)
|
||||
or ""
|
||||
).encode()
|
||||
).hexdigest(),
|
||||
}
|
||||
)
|
||||
|
|
@ -1698,7 +1688,9 @@ def score_live_trial(
|
|||
baseline_receipts = {
|
||||
str(result.get("prompt_id")): _receipt_score(
|
||||
result,
|
||||
require_database_contract=bool(by_prompt.get(str(result.get("prompt_id")), {}).get("requires_database_contract")),
|
||||
require_database_contract=bool(
|
||||
by_prompt.get(str(result.get("prompt_id")), {}).get("requires_database_contract")
|
||||
),
|
||||
require_database_receipt=bool(
|
||||
by_prompt.get(str(result.get("prompt_id")), {}).get("requires_database_receipt")
|
||||
),
|
||||
|
|
@ -1729,9 +1721,7 @@ def score_live_trial(
|
|||
and baseline_receipts.get(prompt["id"], {}).get("pass")
|
||||
)
|
||||
autonomous_prompt_ids = [
|
||||
prompt["id"]
|
||||
for prompt in trial["prompts"]
|
||||
if prompt.get("requires_grounded_retrieval_answer") is True
|
||||
prompt["id"] for prompt in trial["prompts"] if prompt.get("requires_grounded_retrieval_answer") is True
|
||||
]
|
||||
autonomous_rows: list[dict[str, Any]] = []
|
||||
for prompt_id in autonomous_prompt_ids:
|
||||
|
|
@ -1741,15 +1731,9 @@ def score_live_trial(
|
|||
supported_claim_ids = set(grounded_receipt_score.get("supported_claim_ids") or [])
|
||||
supported_source_ids = set(grounded_receipt_score.get("supported_source_ids") or [])
|
||||
supported_identifiers = set(grounded_receipt_score.get("supported_identifiers") or [])
|
||||
ablation_claim_citations = {
|
||||
match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(baseline_reply)
|
||||
}
|
||||
ablation_source_citations = {
|
||||
match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(baseline_reply)
|
||||
}
|
||||
ablation_reply_identifiers = {
|
||||
match.group(0).lower() for match in UUID_RE.finditer(baseline_reply)
|
||||
}
|
||||
ablation_claim_citations = {match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(baseline_reply)}
|
||||
ablation_source_citations = {match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(baseline_reply)}
|
||||
ablation_reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(baseline_reply)}
|
||||
grounded_answer_pass = bool(
|
||||
semantic_by_prompt.get(prompt_id, {}).get("pass")
|
||||
and subject_alignment.get(prompt_id)
|
||||
|
|
@ -1782,9 +1766,7 @@ def score_live_trial(
|
|||
autonomous_prompt_count = len(autonomous_rows)
|
||||
autonomous_grounded_passes = sum(1 for row in autonomous_rows if row["grounded_answer_pass"])
|
||||
autonomous_ablation_passes = sum(1 for row in autonomous_rows if row["ablation_answer_pass"])
|
||||
autonomous_causal_passes = sum(
|
||||
1 for row in autonomous_rows if row["causally_attributed_grounded_pass"]
|
||||
)
|
||||
autonomous_causal_passes = sum(1 for row in autonomous_rows if row["causally_attributed_grounded_pass"])
|
||||
autonomous_retrieval_comparison = {
|
||||
"method": "both_arms_semantic_subject_and_citations_scored_against_grounded_receipted_ids",
|
||||
"prompt_ids": autonomous_prompt_ids,
|
||||
|
|
@ -1797,9 +1779,7 @@ def score_live_trial(
|
|||
if autonomous_prompt_count
|
||||
else 0.0
|
||||
),
|
||||
"identical_reply_prompt_ids": [
|
||||
row["prompt_id"] for row in autonomous_rows if row["replies_identical"]
|
||||
],
|
||||
"identical_reply_prompt_ids": [row["prompt_id"] for row in autonomous_rows if row["replies_identical"]],
|
||||
"rows": autonomous_rows,
|
||||
"pass": bool(
|
||||
autonomous_prompt_count
|
||||
|
|
@ -1812,8 +1792,7 @@ def score_live_trial(
|
|||
prompt["id"] for prompt in trial["prompts"] if prompt.get("requires_tool_evidence_token") is True
|
||||
]
|
||||
grounded_evidence_by_prompt = {
|
||||
item["prompt_id"]: item["evidence_answer_score"]["grounded_tool_semantic_hashes"]
|
||||
for item in prompt_scores
|
||||
item["prompt_id"]: item["evidence_answer_score"]["grounded_tool_semantic_hashes"] for item in prompt_scores
|
||||
}
|
||||
baseline_evidence_scores = {
|
||||
prompt_id: _evidence_answer_score(
|
||||
|
|
@ -1826,9 +1805,7 @@ def score_live_trial(
|
|||
for prompt_id in evidence_prompt_ids
|
||||
}
|
||||
current_evidence_passes = sum(
|
||||
1
|
||||
for item in prompt_scores
|
||||
if item["prompt_id"] in evidence_prompt_ids and item["evidence_answer_pass"] is True
|
||||
1 for item in prompt_scores if item["prompt_id"] in evidence_prompt_ids and item["evidence_answer_pass"] is True
|
||||
)
|
||||
baseline_evidence_passes = sum(1 for item in baseline_evidence_scores.values() if item["pass"] is True)
|
||||
evidence_prompt_count = len(evidence_prompt_ids)
|
||||
|
|
@ -1858,6 +1835,7 @@ def score_live_trial(
|
|||
not item["receipt_score"].get("unsupported_identifiers") for item in prompt_scores
|
||||
),
|
||||
}
|
||||
|
||||
def model_identities(value: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
identities = {
|
||||
canonical_sha256(
|
||||
|
|
@ -1889,7 +1867,9 @@ def score_live_trial(
|
|||
"prompt_set_hash_equal": baseline_report.get("trial_prompt_set_sha256")
|
||||
== report.get("trial_prompt_set_sha256")
|
||||
== trial["prompt_set_sha256"],
|
||||
"source_hashes_equal": baseline_report.get("source_hashes") == report.get("source_hashes") == protocol["source_hashes"],
|
||||
"source_hashes_equal": baseline_report.get("source_hashes")
|
||||
== report.get("source_hashes")
|
||||
== protocol["source_hashes"],
|
||||
"live_behavior_manifest_equal": bool(
|
||||
(baseline_report.get("live_behavior_manifest_before") or {}).get("behavior_sha256")
|
||||
and (baseline_report.get("live_behavior_manifest_before") or {}).get("behavior_sha256")
|
||||
|
|
@ -1911,9 +1891,7 @@ def score_live_trial(
|
|||
}
|
||||
threshold = float(protocol["thresholds"]["minimum_trial_grounded_pass_rate"])
|
||||
evidence_threshold = float(protocol["thresholds"]["minimum_trial_evidence_answer_pass_rate"])
|
||||
evidence_delta_threshold = float(
|
||||
protocol["thresholds"]["minimum_current_minus_ablation_evidence_answer_delta"]
|
||||
)
|
||||
evidence_delta_threshold = float(protocol["thresholds"]["minimum_current_minus_ablation_evidence_answer_delta"])
|
||||
score: dict[str, Any] = {
|
||||
"schema": TRIAL_SCORE_SCHEMA,
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
|
|
@ -2059,8 +2037,7 @@ def validate_trial_score(protocol: dict[str, Any], trial: dict[str, Any], score:
|
|||
"protocol_id_bound": payload.get("protocol_id") == protocol.get("protocol_id"),
|
||||
"protocol_hash_bound": payload.get("protocol_hash_sha256") == protocol.get("protocol_hash_sha256"),
|
||||
"next_trial_id_bound": payload.get("next_trial_id") == trial.get("trial_id"),
|
||||
"next_prompt_set_bound": payload.get("next_trial_prompt_set_sha256")
|
||||
== trial.get("prompt_set_sha256"),
|
||||
"next_prompt_set_bound": payload.get("next_trial_prompt_set_sha256") == trial.get("prompt_set_sha256"),
|
||||
}, payload
|
||||
|
||||
expected_prompt_ids = [item["id"] for item in trial["prompts"]]
|
||||
|
|
@ -2076,9 +2053,7 @@ def validate_trial_score(protocol: dict[str, Any], trial: dict[str, Any], score:
|
|||
baseline_semantic_scores = (
|
||||
baseline.get("semantic_scores") if isinstance(baseline.get("semantic_scores"), list) else []
|
||||
)
|
||||
baseline_semantic_ids = [
|
||||
str(item.get("prompt_id")) for item in baseline_semantic_scores if isinstance(item, dict)
|
||||
]
|
||||
baseline_semantic_ids = [str(item.get("prompt_id")) for item in baseline_semantic_scores if isinstance(item, dict)]
|
||||
baseline_semantic_passes = sum(
|
||||
1 for item in baseline_semantic_scores if isinstance(item, dict) and item.get("pass") is True
|
||||
)
|
||||
|
|
@ -2160,25 +2135,17 @@ def validate_trial_score(protocol: dict[str, Any], trial: dict[str, Any], score:
|
|||
"evidence_prompt_count": evidence.get("prompt_count") == evidence_count,
|
||||
"current_evidence_passes_recomputed": evidence.get("current_passes") == current_evidence_passes,
|
||||
"baseline_evidence_passes_recomputed": evidence.get("ablation_passes") == baseline_evidence_passes,
|
||||
"current_evidence_rate_recomputed": abs(
|
||||
number(evidence.get("current_pass_rate")) - current_evidence_rate
|
||||
)
|
||||
"current_evidence_rate_recomputed": abs(number(evidence.get("current_pass_rate")) - current_evidence_rate)
|
||||
< 1e-12,
|
||||
"baseline_evidence_rate_recomputed": abs(
|
||||
number(evidence.get("ablation_pass_rate")) - baseline_evidence_rate
|
||||
)
|
||||
"baseline_evidence_rate_recomputed": abs(number(evidence.get("ablation_pass_rate")) - baseline_evidence_rate)
|
||||
< 1e-12,
|
||||
"evidence_delta_recomputed": abs(
|
||||
number(evidence.get("current_minus_ablation_delta"))
|
||||
- (current_evidence_rate - baseline_evidence_rate)
|
||||
number(evidence.get("current_minus_ablation_delta")) - (current_evidence_rate - baseline_evidence_rate)
|
||||
)
|
||||
< 1e-12,
|
||||
"grounded_report_artifact_bound": bool(grounded_artifact_checks)
|
||||
and all(grounded_artifact_checks.values()),
|
||||
"baseline_report_artifact_bound": bool(baseline_artifact_checks)
|
||||
and all(baseline_artifact_checks.values()),
|
||||
"restart_receipt_artifact_bound": bool(restart_artifact_checks)
|
||||
and all(restart_artifact_checks.values()),
|
||||
"grounded_report_artifact_bound": bool(grounded_artifact_checks) and all(grounded_artifact_checks.values()),
|
||||
"baseline_report_artifact_bound": bool(baseline_artifact_checks) and all(baseline_artifact_checks.values()),
|
||||
"restart_receipt_artifact_bound": bool(restart_artifact_checks) and all(restart_artifact_checks.values()),
|
||||
"derivation_core_hash_bound": _valid_sha256(score.get("derivation_core_sha256"))
|
||||
and score.get("derivation_core_sha256") == canonical_sha256(stored_core),
|
||||
"score_recomputed_from_retained_artifacts": bool(recomputed)
|
||||
|
|
@ -2197,9 +2164,7 @@ def validate_trial_score(protocol: dict[str, Any], trial: dict[str, Any], score:
|
|||
and all(value is True for value in mapping(score.get("comparison_axis_checks")).values()),
|
||||
"critical_prompt_checks_passed": bool(mapping(score.get("critical_prompt_checks")))
|
||||
and all(value is True for value in mapping(score.get("critical_prompt_checks")).values()),
|
||||
"autonomous_retrieval_comparison_passed": mapping(
|
||||
score.get("autonomous_retrieval_comparison")
|
||||
).get("pass")
|
||||
"autonomous_retrieval_comparison_passed": mapping(score.get("autonomous_retrieval_comparison")).get("pass")
|
||||
is True,
|
||||
"restart_receipt_validation_passed": mapping(score.get("restart_receipt_validation")).get("pass") is True,
|
||||
"score_passed": score.get("pass") is True,
|
||||
|
|
@ -2222,7 +2187,9 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
|
|||
trial_id: validate_trial_score(protocol, trial_by_id[trial_id], by_id.get(trial_id, {}))
|
||||
for trial_id in expected_ids
|
||||
}
|
||||
current_rates = [float(by_id[trial_id].get("grounded_pass_rate") or 0.0) for trial_id in expected_ids if trial_id in by_id]
|
||||
current_rates = [
|
||||
float(by_id[trial_id].get("grounded_pass_rate") or 0.0) for trial_id in expected_ids if trial_id in by_id
|
||||
]
|
||||
baseline_rates = [
|
||||
float((by_id[trial_id].get("receipt_ablation") or {}).get("grounded_pass_rate") or 0.0)
|
||||
for trial_id in expected_ids
|
||||
|
|
@ -2251,9 +2218,7 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
|
|||
]
|
||||
autonomous_retrieval_deltas = [
|
||||
float(
|
||||
(by_id[trial_id].get("autonomous_retrieval_comparison") or {}).get(
|
||||
"grounded_minus_ablation_answer_delta"
|
||||
)
|
||||
(by_id[trial_id].get("autonomous_retrieval_comparison") or {}).get("grounded_minus_ablation_answer_delta")
|
||||
or 0.0
|
||||
)
|
||||
for trial_id in expected_ids
|
||||
|
|
@ -2285,8 +2250,7 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
|
|||
"minimum_non_tautological_evidence_ablation_delta": (mean_evidence_current - mean_evidence_baseline)
|
||||
>= float(thresholds["minimum_current_minus_ablation_evidence_answer_delta"]),
|
||||
"baseline_rejects_all_ungrounded_receipts": all(rate == 0.0 for rate in baseline_rates),
|
||||
"autonomous_retrieval_has_causal_lift_every_trial": len(autonomous_retrieval_deltas)
|
||||
== len(expected_ids)
|
||||
"autonomous_retrieval_has_causal_lift_every_trial": len(autonomous_retrieval_deltas) == len(expected_ids)
|
||||
and all(delta == 1.0 for delta in autonomous_retrieval_deltas),
|
||||
"broad_semantic_comparison_reported": len(semantic_current_rates)
|
||||
== len(semantic_baseline_rates)
|
||||
|
|
@ -2297,6 +2261,16 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
|
|||
if item["session_mode"] == "post_restart_clean_session"
|
||||
),
|
||||
}
|
||||
evaluation_pass = all(checks.values())
|
||||
independent_live_attestation = {
|
||||
"status": "not_configured",
|
||||
"pass": False,
|
||||
"required_for": "machine_verifiable_T3_live_readonly",
|
||||
"reason": (
|
||||
"The protocol, reports, and scores are caller-readable artifacts. They validate benchmark semantics "
|
||||
"but do not contain a non-caller-mintable platform or service attestation of live execution."
|
||||
),
|
||||
}
|
||||
aggregate = {
|
||||
"schema": AGGREGATE_SCHEMA,
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
|
|
@ -2318,9 +2292,7 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
|
|||
"mean_ablation_evidence_answer_pass_rate": mean_evidence_baseline,
|
||||
"current_minus_ablation_evidence_answer_delta": mean_evidence_current - mean_evidence_baseline,
|
||||
"current_evidence_answer_pass_rate_population_stddev": evidence_stddev,
|
||||
"minimum_current_evidence_answer_pass_rate": min(evidence_current_rates)
|
||||
if evidence_current_rates
|
||||
else 0.0,
|
||||
"minimum_current_evidence_answer_pass_rate": min(evidence_current_rates) if evidence_current_rates else 0.0,
|
||||
"current_semantic_pass_rates": semantic_current_rates,
|
||||
"ablation_semantic_pass_rates": semantic_baseline_rates,
|
||||
"mean_current_semantic_pass_rate": mean_semantic_current,
|
||||
|
|
@ -2331,16 +2303,19 @@ def aggregate_trial_scores(protocol: dict[str, Any], trial_scores: list[dict[str
|
|||
"trial_score_integrity": integrity,
|
||||
"trial_scores": trial_scores,
|
||||
"required_tier": "T3_live_readonly",
|
||||
"current_tier": "T2_runtime_artifact_validation",
|
||||
"operator_observed_tier": "T3_live_readonly_unattested",
|
||||
"evaluation_pass": evaluation_pass,
|
||||
"independent_live_attestation": independent_live_attestation,
|
||||
"machine_verifiable_t3_pass": evaluation_pass and independent_live_attestation["pass"],
|
||||
"claim_ceiling": (
|
||||
"Repeated live VPS GatewayRunner reasoning with frozen blinded families and a current-build no-DB "
|
||||
"ablation. Both arms are scored against the grounded arm's model-visible receipt tokens, so identical "
|
||||
"answers cannot create the gated evidence delta. Broad semantic rates for both arms are reported "
|
||||
"separately and are descriptive rather than silently inferred from missing receipts. Complete "
|
||||
"tool/execution receipts and unchanged canonical database fingerprints are retained. No Telegram "
|
||||
"delivery or production apply is proven."
|
||||
"The retained artifacts can validate frozen-family scoring, grounded-versus-ablated comparisons, "
|
||||
"tool/response binding, and unchanged database fingerprints at T2. An operator may separately report "
|
||||
"that they were captured from a live VPS, but these caller-readable files do not independently prove "
|
||||
"that T3 origin. No Telegram delivery or production apply is proven."
|
||||
),
|
||||
}
|
||||
aggregate["pass"] = all(checks.values())
|
||||
aggregate["pass"] = aggregate["machine_verifiable_t3_pass"]
|
||||
return aggregate
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ def test_plugin_injects_bounded_retrieval_rows_and_writes_body_redacted_trace(tm
|
|||
assert "NEVER_INCLUDE_RAW_UNUSED_FIELD" not in context
|
||||
assert len(context) < 8_000
|
||||
injected_hash_match = re.search(r'injected_rows_sha256="([0-9a-f]{64})"', context)
|
||||
injected_payload_match = re.search(r'\n(\{[^\n]+\})\n</leo_retrieved_database_rows>', context)
|
||||
injected_payload_match = re.search(r"\n(\{[^\n]+\})\n</leo_retrieved_database_rows>", context)
|
||||
assert injected_hash_match is not None
|
||||
assert injected_payload_match is not None
|
||||
injected_rows_sha256 = injected_hash_match.group(1)
|
||||
|
|
@ -275,9 +275,7 @@ def test_plugin_compacts_generic_over_budget_reply_without_a_query_template(tmp_
|
|||
"What should we challenge first?"
|
||||
)
|
||||
|
||||
result = module._post_llm_call(
|
||||
session_id="budget-session", user_message=query, assistant_response=draft
|
||||
)
|
||||
result = module._post_llm_call(session_id="budget-session", user_message=query, assistant_response=draft)
|
||||
|
||||
assert result is not None
|
||||
delivered = result["assistant_response"]
|
||||
|
|
@ -320,9 +318,7 @@ def test_broad_report_learning_question_requires_database_truth() -> None:
|
|||
assert module._requires_database_truth(query) is True
|
||||
|
||||
|
||||
def test_plugin_preserves_same_session_chat_only_memory_set_and_recall_responses(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
def test_plugin_preserves_same_session_chat_only_memory_set_and_recall_responses(tmp_path: Path, monkeypatch) -> None:
|
||||
module = load_plugin()
|
||||
trace = tmp_path / "trace.jsonl"
|
||||
monkeypatch.setenv("LEO_DB_CONTEXT_TRACE_PATH", str(trace))
|
||||
|
|
@ -478,8 +474,7 @@ def test_plugin_requires_named_proposal_receipt_when_live_match_exists() -> None
|
|||
valid = (
|
||||
"No.\n"
|
||||
"DB readback: proposal: 10bc0719-1c2b-5f42-a41e-86e6478692cb; status: pending_review; "
|
||||
"applied_at: none; readiness: needs_human_review.\n"
|
||||
+ common
|
||||
"applied_at: none; readiness: needs_human_review.\n" + common
|
||||
)
|
||||
|
||||
missing_receipt_issues = module.response_contract_issues("No.\n" + common, contracts)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ def test_extracts_completed_read_only_context_call_and_receipt() -> None:
|
|||
"id": "call-secret-looking-id",
|
||||
"function": {
|
||||
"name": "terminal",
|
||||
"arguments": '{"command":"teleo-kb context \\\"AI sandbagging\\\""}',
|
||||
"arguments": '{"command":"teleo-kb context \\"AI sandbagging\\""}',
|
||||
},
|
||||
}
|
||||
],
|
||||
|
|
@ -131,7 +131,7 @@ def test_unmatched_or_failed_result_does_not_prove_database_call() -> None:
|
|||
"role": "tool",
|
||||
"tool_call_id": "call-missing",
|
||||
"content": "Traceback (most recent call last): connection failed",
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
assert extract_kb_tool_trace(unmatched)["database_tool_call_proven"] is False
|
||||
|
|
|
|||
|
|
@ -99,9 +99,7 @@ def test_safety_gate_passes_only_for_complete_no_send_no_write_run() -> None:
|
|||
def test_safety_gate_rejects_write_capable_tool_and_incomplete_receipt() -> None:
|
||||
report = safe_report()
|
||||
report["results"][0]["database_tool_trace"]["database_tool_calls_read_only"] = False
|
||||
report["results"][0]["database_tool_trace"]["calls"][0]["database_invocations"][0][
|
||||
"access_mode"
|
||||
] = "write"
|
||||
report["results"][0]["database_tool_trace"]["calls"][0]["database_invocations"][0]["access_mode"] = "write"
|
||||
report["execution_manifest_summary"]["all_turns_attribution_complete"] = False
|
||||
|
||||
gate = suite.build_safety_gate(report)
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ def test_guarded_remote_script_compiles_and_installs_preexecution_gate(grounding
|
|||
assert "timeout=180" in script
|
||||
assert "timeout=300" not in script
|
||||
if grounding_mode == "db_tool_ablated":
|
||||
assert 'shutil.rmtree(db_context_dir)' in script
|
||||
assert "shutil.rmtree(db_context_dir)" in script
|
||||
|
||||
|
||||
def test_prompt_leakage_scan_uses_partial_markers_and_runtime_roots(
|
||||
|
|
@ -82,6 +82,44 @@ def test_current_repo_runtime_has_no_frozen_prompt_marker_leakage() -> None:
|
|||
assert scan["scanned_files"] > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("root_state", ("missing", "empty", "symlink"))
|
||||
def test_prompt_leakage_scan_fails_closed_without_real_coverage(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, root_state: str
|
||||
) -> None:
|
||||
runtime_root = tmp_path / "runtime"
|
||||
if root_state == "empty":
|
||||
runtime_root.mkdir()
|
||||
elif root_state == "symlink":
|
||||
target = tmp_path / "target"
|
||||
target.mkdir()
|
||||
(target / "runtime.py").write_text("safe runtime text", encoding="utf-8")
|
||||
runtime_root.symlink_to(target, target_is_directory=True)
|
||||
monkeypatch.setattr(suite, "RUNTIME_PROMPT_SCAN_ROOTS", (runtime_root,))
|
||||
|
||||
scan = suite.prompt_leakage_scan(protocol())
|
||||
|
||||
assert scan["pass"] is False
|
||||
assert scan["expected_roots"][0]["exists"] is (root_state != "missing")
|
||||
assert scan["scanned_files"] == 0
|
||||
|
||||
|
||||
def test_prompt_leakage_scan_fails_closed_on_nested_symlink(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
runtime_root = tmp_path / "runtime"
|
||||
runtime_root.mkdir()
|
||||
(runtime_root / "runtime.py").write_text("safe runtime text", encoding="utf-8")
|
||||
hidden = tmp_path / "hidden"
|
||||
hidden.mkdir()
|
||||
(hidden / "prompt.md").write_text("unscanned prompt text", encoding="utf-8")
|
||||
(runtime_root / "linked").symlink_to(hidden, target_is_directory=True)
|
||||
monkeypatch.setattr(suite, "RUNTIME_PROMPT_SCAN_ROOTS", (runtime_root,))
|
||||
|
||||
scan = suite.prompt_leakage_scan(protocol())
|
||||
|
||||
assert scan["pass"] is False
|
||||
assert scan["scanned_files"] == 1
|
||||
assert len(scan["symlink_entries"]) == 1
|
||||
|
||||
|
||||
def test_restart_probe_requires_full_fingerprint_guard_and_cleanup() -> None:
|
||||
counts = {"public.claims": 2}
|
||||
fingerprint = {"status": "ok", "fingerprint_sha256": "a" * 64}
|
||||
|
|
|
|||
|
|
@ -166,9 +166,7 @@ def test_oos_score_passes_complete_behavior_and_memory_pair() -> None:
|
|||
assert score["pass"] is True
|
||||
assert score["passes"] == 16
|
||||
assert score["memory_continuity"]["same_blocker_recalled"] is True
|
||||
assert {"approved", "applied", "canonical"} <= set(
|
||||
score["memory_continuity"]["recall_overlap_terms"]
|
||||
)
|
||||
assert {"approved", "applied", "canonical"} <= set(score["memory_continuity"]["recall_overlap_terms"])
|
||||
|
||||
|
||||
def test_oos_autonomous_retrieval_semantics_reject_generic_challenge() -> None:
|
||||
|
|
@ -559,9 +557,7 @@ def test_oos_composition_rejects_belief_update_in_supported_apply_list() -> None
|
|||
"(claims, sources, evidence, edges, reasoning_tools, belief updates). "
|
||||
"behavioral_rules and governance_gates remain staged for a separate reviewed apply capability."
|
||||
)
|
||||
assert benchmark.composition_capability_issues("OOS-06", reply) == [
|
||||
"unsupported_collection_listed_as_applyable"
|
||||
]
|
||||
assert benchmark.composition_capability_issues("OOS-06", reply) == ["unsupported_collection_listed_as_applyable"]
|
||||
|
||||
|
||||
def test_db_compiled_responses_pass_composition_and_source_intake_benchmarks() -> None:
|
||||
|
|
|
|||
|
|
@ -83,9 +83,7 @@ def retrieval_receipt(query_sha256: str) -> dict:
|
|||
|
||||
def rehash_retrieval_receipt(receipt: dict) -> None:
|
||||
trace_payload = {
|
||||
key: value
|
||||
for key, value in receipt.items()
|
||||
if key not in {"receipt_sha256", "trace_payload_sha256"}
|
||||
key: value for key, value in receipt.items() if key not in {"receipt_sha256", "trace_payload_sha256"}
|
||||
}
|
||||
receipt["trace_payload_sha256"] = hashlib.sha256(
|
||||
json.dumps(trace_payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
|
|
@ -289,9 +287,7 @@ def attach_fake_execution_manifests(
|
|||
}
|
||||
previous_execution_sha256 = None
|
||||
allowed_missing = (
|
||||
protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING
|
||||
if grounded
|
||||
else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING
|
||||
protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING if grounded else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING
|
||||
)
|
||||
fingerprint = report["db_fingerprint_before"]
|
||||
counts_sha256 = protocol_lib.canonical_sha256({})
|
||||
|
|
@ -319,9 +315,7 @@ def attach_fake_execution_manifests(
|
|||
"runtime": {
|
||||
"behavior_sha256": report["executed_behavior_manifest"]["behavior_sha256"],
|
||||
"hermes_runtime": report["executed_behavior_manifest"]["hermes_runtime"],
|
||||
"teleo_infrastructure_runtime": report["executed_behavior_manifest"][
|
||||
"teleo_infrastructure_runtime"
|
||||
],
|
||||
"teleo_infrastructure_runtime": report["executed_behavior_manifest"]["teleo_infrastructure_runtime"],
|
||||
"harness_source": harness_source,
|
||||
},
|
||||
"model_execution": {
|
||||
|
|
@ -475,8 +469,7 @@ def fake_report(protocol: dict, trial: dict, *, grounded: bool) -> dict:
|
|||
"remote_temp_profile_prompt_leakage_scan": {
|
||||
"scope": "full_model_visible_temp_profile_excluding_sessions_state_memories_and_venv",
|
||||
"expected_roots": [
|
||||
{"name": name, "exists": True, "is_symlink": False}
|
||||
for name in ("profile", "skills", "plugins", "bin")
|
||||
{"name": name, "exists": True, "is_symlink": False} for name in ("profile", "skills", "plugins", "bin")
|
||||
],
|
||||
"scanned_files": 10,
|
||||
"scanned_bytes": 1000,
|
||||
|
|
@ -520,9 +513,7 @@ def restart_receipt(protocol: dict, directory: Path, trial: dict) -> dict:
|
|||
|
||||
def probe(path: Path, *, before_restart: bool) -> dict:
|
||||
payload = {
|
||||
"generated_at_utc": "2026-07-15T00:00:00+00:00"
|
||||
if before_restart
|
||||
else "2026-07-15T00:01:15+00:00",
|
||||
"generated_at_utc": "2026-07-15T00:00:00+00:00" if before_restart else "2026-07-15T00:01:15+00:00",
|
||||
"protocol_id": protocol["protocol_id"],
|
||||
"protocol_hash_sha256": protocol["protocol_hash_sha256"],
|
||||
"source_hashes": protocol["source_hashes"],
|
||||
|
|
@ -633,19 +624,14 @@ def test_protocol_freezes_three_unique_variants_and_follow_up_shapes() -> None:
|
|||
assert protocol["trials"][-1]["session_mode"] == "post_restart_clean_session"
|
||||
for family_id in protocol["blinding"]["prompt_families"]:
|
||||
prompts = [
|
||||
next(item for item in trial["prompts"] if item["family_id"] == family_id)
|
||||
for trial in protocol["trials"]
|
||||
next(item for item in trial["prompts"] if item["family_id"] == family_id) for trial in protocol["trials"]
|
||||
]
|
||||
assert len({item["variant_index"] for item in prompts}) == 3
|
||||
assert len({item["expected_follow_up"] for item in prompts}) == 3
|
||||
assert all(item["leakage_markers"] for item in prompts)
|
||||
assert all("row id:" not in item["message"].lower() for item in prompts)
|
||||
retrieval_prompts = [
|
||||
next(
|
||||
item
|
||||
for item in trial["prompts"]
|
||||
if item["family_id"] == "autonomous_retrieval_reasoning"
|
||||
)
|
||||
next(item for item in trial["prompts"] if item["family_id"] == "autonomous_retrieval_reasoning")
|
||||
for trial in protocol["trials"]
|
||||
]
|
||||
assert len({item["subject"] for item in retrieval_prompts}) == 3
|
||||
|
|
@ -669,9 +655,10 @@ def test_protocol_validation_rejects_prompt_and_source_hash_tampering() -> None:
|
|||
protocol["protocol_hash_sha256"] = protocol_lib.canonical_sha256(
|
||||
{key: value for key, value in protocol.items() if key != "protocol_hash_sha256"}
|
||||
)
|
||||
assert "source_changed_after_freeze:readonly_guard_sha256" in protocol_lib.validate_protocol(
|
||||
protocol, verify_source_hashes=True
|
||||
)["issues"]
|
||||
assert (
|
||||
"source_changed_after_freeze:readonly_guard_sha256"
|
||||
in protocol_lib.validate_protocol(protocol, verify_source_hashes=True)["issues"]
|
||||
)
|
||||
|
||||
|
||||
def test_live_trial_requires_semantics_subject_receipts_and_real_ablation() -> None:
|
||||
|
|
@ -718,11 +705,7 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
|
|||
|
||||
manifest = report["results"][0]["execution_manifest"]
|
||||
manifest["attribution"]["missing_required_bindings"].append("unexpected_gap")
|
||||
stable = {
|
||||
key: value
|
||||
for key, value in manifest.items()
|
||||
if key not in {"generated_at_utc", "execution_sha256"}
|
||||
}
|
||||
stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
|
||||
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
|
||||
validation = protocol_lib._benchmark_execution_chain(
|
||||
report,
|
||||
|
|
@ -734,11 +717,7 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
|
|||
ablated = fake_report(protocol, trial, grounded=False)
|
||||
manifest = ablated["results"][-1]["execution_manifest"]
|
||||
manifest["attribution"]["missing_required_bindings"].append("database_tool_results")
|
||||
stable = {
|
||||
key: value
|
||||
for key, value in manifest.items()
|
||||
if key not in {"generated_at_utc", "execution_sha256"}
|
||||
}
|
||||
stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
|
||||
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
|
||||
validation = protocol_lib._benchmark_execution_chain(
|
||||
ablated,
|
||||
|
|
@ -786,22 +765,14 @@ def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal
|
|||
for result in baseline["results"]:
|
||||
manifest = result["execution_manifest"]
|
||||
manifest["runtime"]["behavior_sha256"] = behavior["behavior_sha256"]
|
||||
stable = {
|
||||
key: value
|
||||
for key, value in manifest.items()
|
||||
if key not in {"generated_at_utc", "execution_sha256"}
|
||||
}
|
||||
stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
|
||||
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
|
||||
for previous, result in zip(baseline["results"], baseline["results"][1:], strict=False):
|
||||
manifest = result["execution_manifest"]
|
||||
manifest["session_boundary"]["conversation"]["previous_execution_sha256"] = previous[
|
||||
"execution_manifest"
|
||||
]["execution_sha256"]
|
||||
stable = {
|
||||
key: value
|
||||
for key, value in manifest.items()
|
||||
if key not in {"generated_at_utc", "execution_sha256"}
|
||||
}
|
||||
manifest["session_boundary"]["conversation"]["previous_execution_sha256"] = previous["execution_manifest"][
|
||||
"execution_sha256"
|
||||
]
|
||||
stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
|
||||
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
|
||||
|
||||
score = protocol_lib.score_live_trial(
|
||||
|
|
@ -812,10 +783,7 @@ def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal
|
|||
)
|
||||
assert score["pass"] is False
|
||||
assert score["executed_behavior_ablation"]["checks"]["non_middleware_components_equal"] is False
|
||||
assert (
|
||||
score["comparison_axis_checks"]["executed_behavior_manifests_capture_only_declared_ablation"]
|
||||
is False
|
||||
)
|
||||
assert score["comparison_axis_checks"]["executed_behavior_manifests_capture_only_declared_ablation"] is False
|
||||
|
||||
|
||||
def test_relative_retained_paths_resolve_from_repo_root() -> None:
|
||||
|
|
@ -848,9 +816,7 @@ def test_live_trial_rejects_duplicate_results_malformed_receipts_and_invented_id
|
|||
first = score["prompt_scores"][0]
|
||||
assert score["pass"] is False
|
||||
assert first["receipt_pass"] is False
|
||||
assert first["receipt_score"]["unsupported_identifiers"] == [
|
||||
"12345678-1234-4123-8123-123456789abc"
|
||||
]
|
||||
assert first["receipt_score"]["unsupported_identifiers"] == ["12345678-1234-4123-8123-123456789abc"]
|
||||
|
||||
grounded = fake_report(protocol, trial, grounded=True)
|
||||
traced_receipt = grounded["results"][0]["database_context_trace"][0]["retrieval_receipt"]
|
||||
|
|
@ -895,11 +861,7 @@ def test_live_trial_rejects_duplicate_results_malformed_receipts_and_invented_id
|
|||
def test_autonomous_retrieval_requires_nonempty_rows_and_receipted_reply_ids() -> None:
|
||||
protocol = frozen_protocol()
|
||||
trial = protocol["trials"][0]
|
||||
prompt = next(
|
||||
item
|
||||
for item in trial["prompts"]
|
||||
if item["family_id"] == "autonomous_retrieval_reasoning"
|
||||
)
|
||||
prompt = next(item for item in trial["prompts"] if item["family_id"] == "autonomous_retrieval_reasoning")
|
||||
|
||||
grounded = fake_report(protocol, trial, grounded=True)
|
||||
result = next(item for item in grounded["results"] if item["prompt_id"] == prompt["id"])
|
||||
|
|
@ -993,11 +955,7 @@ def test_autonomous_retrieval_requires_nonempty_rows_and_receipted_reply_ids() -
|
|||
def test_autonomous_retrieval_ablation_rejects_generic_prose_and_invented_ids() -> None:
|
||||
protocol = frozen_protocol()
|
||||
trial = protocol["trials"][0]
|
||||
prompt = next(
|
||||
item
|
||||
for item in trial["prompts"]
|
||||
if item["family_id"] == "autonomous_retrieval_reasoning"
|
||||
)
|
||||
prompt = next(item for item in trial["prompts"] if item["family_id"] == "autonomous_retrieval_reasoning")
|
||||
grounded = fake_report(protocol, trial, grounded=True)
|
||||
ablated = fake_report(protocol, trial, grounded=False)
|
||||
result = next(item for item in ablated["results"] if item["prompt_id"] == prompt["id"])
|
||||
|
|
@ -1010,9 +968,7 @@ def test_autonomous_retrieval_ablation_rejects_generic_prose_and_invented_ids()
|
|||
"Challenge: the evidence does not prove the broader leap. "
|
||||
"Narrower revision: only the generic source is supported; no mutation was made."
|
||||
)
|
||||
result["execution_manifest"]["turn"]["reply_sha256"] = hashlib.sha256(
|
||||
result["reply"].encode()
|
||||
).hexdigest()
|
||||
result["execution_manifest"]["turn"]["reply_sha256"] = hashlib.sha256(result["reply"].encode()).hexdigest()
|
||||
result["execution_manifest"]["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(
|
||||
{
|
||||
key: value
|
||||
|
|
@ -1064,9 +1020,7 @@ def test_hash_bound_contract_row_ids_are_supported_but_unreceipted_ids_fail() ->
|
|||
|
||||
def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words() -> None:
|
||||
protocol = frozen_protocol()
|
||||
source_prompt = next(
|
||||
item for item in protocol["trials"][0]["prompts"] if item["family_id"] == "source_evidence"
|
||||
)
|
||||
source_prompt = next(item for item in protocol["trials"][0]["prompts"] if item["family_id"] == "source_evidence")
|
||||
wrong_subject = "The database proposal is approved but not applied, so wait for a canonical receipt."
|
||||
assert protocol_lib._subject_alignment(source_prompt, wrong_subject) is False
|
||||
generic_family_only = "The source document has claim_evidence and evidence, but the named packet is omitted."
|
||||
|
|
@ -1077,15 +1031,11 @@ def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words
|
|||
)
|
||||
assert protocol_lib._subject_alignment(source_prompt, duplicate_subject) is False
|
||||
sibling_subject = next(item for item in source_prompt["family_subjects"] if item != source_prompt["subject"])
|
||||
shotgun = (
|
||||
f"{source_prompt['subject']} and {sibling_subject} both use source evidence and claim_evidence."
|
||||
)
|
||||
shotgun = f"{source_prompt['subject']} and {sibling_subject} both use source evidence and claim_evidence."
|
||||
assert protocol_lib._subject_alignment(source_prompt, shotgun) is False
|
||||
|
||||
retrieval_prompt = next(
|
||||
item
|
||||
for item in protocol["trials"][0]["prompts"]
|
||||
if item["family_id"] == "autonomous_retrieval_reasoning"
|
||||
item for item in protocol["trials"][0]["prompts"] if item["family_id"] == "autonomous_retrieval_reasoning"
|
||||
)
|
||||
good = (
|
||||
f"{retrieval_prompt['subject']}. Claim body and source evidence are distinct. "
|
||||
|
|
@ -1230,15 +1180,24 @@ def test_receipt_discriminator_rejects_wrong_command_extra_call_and_wrong_token(
|
|||
|
||||
def test_aggregate_recomputes_integrity_and_rejects_forged_minimal_scores(tmp_path: Path) -> None:
|
||||
protocol = frozen_protocol()
|
||||
scores = [
|
||||
score_for_trial(protocol, trial, tmp_path / trial["trial_id"])
|
||||
for trial in protocol["trials"]
|
||||
]
|
||||
scores = [score_for_trial(protocol, trial, tmp_path / trial["trial_id"]) for trial in protocol["trials"]]
|
||||
aggregate = protocol_lib.aggregate_trial_scores(protocol, scores)
|
||||
assert aggregate["pass"] is True
|
||||
assert aggregate["evaluation_pass"] is True
|
||||
assert aggregate["machine_verifiable_t3_pass"] is False
|
||||
assert aggregate["pass"] is False
|
||||
assert aggregate["current_tier"] == "T2_runtime_artifact_validation"
|
||||
assert aggregate["independent_live_attestation"]["pass"] is False
|
||||
assert aggregate["mean_current_grounded_pass_rate"] == 1.0
|
||||
assert aggregate["mean_ablation_grounded_pass_rate"] == 0.0
|
||||
|
||||
caller_attested = copy.deepcopy(scores)
|
||||
for score in caller_attested:
|
||||
score["independent_live_attestation"] = {"pass": True, "issuer": "caller"}
|
||||
rejected_attestation = protocol_lib.aggregate_trial_scores(protocol, caller_attested)
|
||||
assert rejected_attestation["evaluation_pass"] is False
|
||||
assert rejected_attestation["machine_verifiable_t3_pass"] is False
|
||||
assert rejected_attestation["pass"] is False
|
||||
|
||||
forged = [
|
||||
{
|
||||
"trial_id": trial["trial_id"],
|
||||
|
|
@ -1258,8 +1217,7 @@ def test_aggregate_recomputes_integrity_and_rejects_forged_minimal_scores(tmp_pa
|
|||
rejected = protocol_lib.aggregate_trial_scores(protocol, tampered)
|
||||
assert rejected["pass"] is False
|
||||
assert (
|
||||
rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]]["checks"]
|
||||
["comparison_axis_checks_passed"]
|
||||
rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]]["checks"]["comparison_axis_checks_passed"]
|
||||
is False
|
||||
)
|
||||
|
||||
|
|
@ -1269,9 +1227,7 @@ def test_aggregate_recomputes_integrity_and_rejects_forged_minimal_scores(tmp_pa
|
|||
failing_report["results"][0]["reply"] = "This answer is unrelated and ungrounded."
|
||||
report_path.write_text(json.dumps(failing_report, sort_keys=True) + "\n", encoding="utf-8")
|
||||
forged_from_failing_report[0]["grounded_report_sha256"] = hashlib.sha256(report_path.read_bytes()).hexdigest()
|
||||
forged_from_failing_report[0]["grounded_report_payload_sha256"] = protocol_lib.canonical_sha256(
|
||||
failing_report
|
||||
)
|
||||
forged_from_failing_report[0]["grounded_report_payload_sha256"] = protocol_lib.canonical_sha256(failing_report)
|
||||
forged_from_failing_report[0]["derivation_core_sha256"] = protocol_lib.canonical_sha256(
|
||||
protocol_lib.score_derivation_core(forged_from_failing_report[0])
|
||||
)
|
||||
|
|
@ -1285,7 +1241,6 @@ def test_aggregate_recomputes_integrity_and_rejects_forged_minimal_scores(tmp_pa
|
|||
rejected = protocol_lib.aggregate_trial_scores(protocol, forged_from_failing_report)
|
||||
assert rejected["pass"] is False
|
||||
assert (
|
||||
rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]]["checks"]
|
||||
["grounded_report_artifact_bound"]
|
||||
rejected["trial_score_integrity"][protocol["trials"][0]["trial_id"]]["checks"]["grounded_report_artifact_bound"]
|
||||
is False
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue