1003 lines
43 KiB
Python
Executable file
1003 lines
43 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Run and score the read-only m3taversal out-of-sample suite on the live VPS handler."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import leo_oos_readonly_guard as readonly_guard
|
|
import run_leo_direct_claim_handler_suite as handler
|
|
import working_leo_m3taversal_oos_benchmark as benchmark
|
|
import working_leo_m3taversal_oos_protocol as protocol_lib
|
|
|
|
BASE_HANDLER_SCRIPT_BUILDER = handler.build_remote_script
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
|
|
RESULTS_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-current.json"
|
|
SCORE_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.json"
|
|
SCORE_MARKDOWN = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.md"
|
|
PROTOCOL_REPORT_DIR = ROOT / "docs" / "reports" / "leo-oos-reasoning-benchmark-20260715"
|
|
DEFAULT_PROTOCOL_JSON = PROTOCOL_REPORT_DIR / "blinded-protocol.json"
|
|
GROUNDING_MODES = ("grounded", "db_tool_ablated")
|
|
RUNTIME_PROMPT_SCAN_ROOTS = (
|
|
ROOT / "hermes-agent" / "leoclean-skills",
|
|
ROOT / "hermes-agent" / "leoclean-plugins",
|
|
ROOT / "hermes-agent" / "leoclean-bin",
|
|
)
|
|
|
|
|
|
def prompt_leakage_scan(protocol: dict[str, Any]) -> dict[str, Any]:
|
|
"""Fail if a frozen prompt was copied into a runtime skill/plugin/tool."""
|
|
|
|
prompt_markers = [
|
|
(str(prompt["id"]), str(marker))
|
|
for trial in protocol.get("trials") or []
|
|
for prompt in trial.get("prompts") or []
|
|
for marker in prompt.get("leakage_markers") or []
|
|
]
|
|
matches: list[dict[str, str]] = []
|
|
scanned_files = 0
|
|
for root in RUNTIME_PROMPT_SCAN_ROOTS:
|
|
if not root.exists():
|
|
continue
|
|
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
|
try:
|
|
text = path.read_text(encoding="utf-8")
|
|
except (OSError, UnicodeDecodeError):
|
|
continue
|
|
scanned_files += 1
|
|
normalized = " ".join(re.findall(r"[a-z0-9_]+", text.lower()))
|
|
for prompt_id, marker in prompt_markers:
|
|
if marker in normalized:
|
|
display_path = str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path)
|
|
matches.append(
|
|
{"prompt_id": prompt_id, "marker_sha256": hashlib.sha256(marker.encode()).hexdigest(), "path": display_path}
|
|
)
|
|
return {
|
|
"scanned_roots": [str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path) for path in RUNTIME_PROMPT_SCAN_ROOTS],
|
|
"scanned_files": scanned_files,
|
|
"exact_prompt_matches": matches,
|
|
"pass": not matches,
|
|
}
|
|
|
|
|
|
def build_guarded_remote_script(
|
|
run_id: str,
|
|
*,
|
|
prompts: list[dict[str, Any]],
|
|
suite_mode: str,
|
|
report_prefix: str,
|
|
prompt_note: str,
|
|
grounding_mode: str,
|
|
leakage_scan: dict[str, Any],
|
|
) -> str:
|
|
"""Inject the reviewed fail-closed tool surface before GatewayRunner starts."""
|
|
|
|
if grounding_mode not in GROUNDING_MODES:
|
|
raise ValueError(f"unsupported grounding mode: {grounding_mode}")
|
|
script = BASE_HANDLER_SCRIPT_BUILDER(
|
|
run_id,
|
|
prompts=prompts,
|
|
suite_mode=suite_mode,
|
|
report_prefix=report_prefix,
|
|
prompt_note=prompt_note,
|
|
)
|
|
original_plugin_source = handler.DB_CONTEXT_PLUGIN.read_text(encoding="utf-8")
|
|
instrumented_plugin_source = protocol_lib.instrument_db_context_plugin_source(original_plugin_source)
|
|
encoded_original_plugin_source = json.dumps(original_plugin_source)
|
|
if script.count(encoded_original_plugin_source) != 1:
|
|
raise RuntimeError("embedded DB context plugin source marker changed")
|
|
script = script.replace(
|
|
encoded_original_plugin_source,
|
|
json.dumps(instrumented_plugin_source),
|
|
1,
|
|
)
|
|
guard_source = Path(readonly_guard.__file__).resolve().read_text(encoding="utf-8")
|
|
function_marker = "async def run_suite():"
|
|
if script.count(function_marker) != 1:
|
|
raise RuntimeError("remote harness run_suite marker changed")
|
|
script = script.replace(
|
|
function_marker,
|
|
"OOS_READONLY_GUARD_SOURCE = "
|
|
+ json.dumps(guard_source)
|
|
+ "\nOOS_GROUNDING_MODE = "
|
|
+ json.dumps(grounding_mode)
|
|
+ "\nOOS_GUARD_SOURCE_SHA256 = "
|
|
+ json.dumps(hashlib.sha256(guard_source.encode()).hexdigest())
|
|
+ "\nOOS_PROMPT_LEAKAGE_SCAN = "
|
|
+ json.dumps(leakage_scan)
|
|
+ "\n\n"
|
|
+ function_marker,
|
|
)
|
|
gateway_marker = " from gateway.session import SessionSource\n\n source = SessionSource("
|
|
if script.count(gateway_marker) != 1:
|
|
raise RuntimeError("remote harness GatewayRunner marker changed")
|
|
injection = """ import re
|
|
from gateway.session import SessionSource
|
|
from gateway.platforms.telegram import TelegramAdapter
|
|
|
|
db_context_dir = temp_profile / "plugins" / "leo-db-context"
|
|
if OOS_GROUNDING_MODE == "db_tool_ablated":
|
|
if db_context_dir.is_symlink():
|
|
db_context_dir.unlink()
|
|
elif db_context_dir.exists():
|
|
shutil.rmtree(db_context_dir)
|
|
guard_module = types.ModuleType("leo_oos_readonly_guard_embedded")
|
|
guard_module.__file__ = "<embedded leo_oos_readonly_guard.py>"
|
|
exec(compile(OOS_READONLY_GUARD_SOURCE, guard_module.__file__, "exec"), guard_module.__dict__)
|
|
tool_surface = guard_module.install_read_only_tool_surface(
|
|
temp_profile,
|
|
allow_kb_reads=OOS_GROUNDING_MODE == "grounded",
|
|
)
|
|
report["executed_behavior_manifest"] = build_behavior_manifest(temp_profile)
|
|
telegram_transport_attempts = []
|
|
telegram_outbound_methods = (
|
|
"_send_with_retry",
|
|
"edit_message",
|
|
"play_tts",
|
|
"send",
|
|
"send_animation",
|
|
"send_document",
|
|
"send_image",
|
|
"send_image_file",
|
|
"send_model_picker",
|
|
"send_typing",
|
|
"send_update_prompt",
|
|
"send_video",
|
|
"send_voice",
|
|
)
|
|
patched_telegram_methods = []
|
|
def make_transport_deny(method_name):
|
|
async def deny_transport(*args, **kwargs):
|
|
telegram_transport_attempts.append({
|
|
"method": method_name,
|
|
"positional_argument_count": len(args),
|
|
"keyword_names": sorted(str(key) for key in kwargs),
|
|
})
|
|
report["telegram_transport_deny"]["attempt_count"] = len(telegram_transport_attempts)
|
|
report["telegram_transport_deny"]["attempts"] = list(telegram_transport_attempts)
|
|
write_report_snapshot(report)
|
|
raise RuntimeError("Telegram transport is denied in the OOS no-post harness")
|
|
return deny_transport
|
|
for method_name in telegram_outbound_methods:
|
|
if hasattr(TelegramAdapter, method_name):
|
|
setattr(TelegramAdapter, method_name, make_transport_deny(method_name))
|
|
patched_telegram_methods.append(method_name)
|
|
|
|
remote_leakage_matches = []
|
|
remote_scanned_files = 0
|
|
remote_scanned_bytes = 0
|
|
remote_scan_errors = []
|
|
remote_symlink_targets = 0
|
|
prompt_markers = []
|
|
for prompt in PROMPTS:
|
|
words = re.findall(r"[a-z0-9_]+", str(prompt.get("message") or "").lower())
|
|
width = 16
|
|
starts = (0,) if len(words) <= width else (0, max(0, (len(words) - width) // 2), len(words) - width)
|
|
for start in starts:
|
|
marker = " ".join(words[start:start + width])
|
|
if marker:
|
|
prompt_markers.append((str(prompt.get("id") or ""), marker))
|
|
remote_scan_roots = {
|
|
"profile": temp_profile,
|
|
"skills": temp_profile / "skills",
|
|
"plugins": temp_profile / "plugins",
|
|
"bin": temp_profile / "bin",
|
|
}
|
|
excluded_profile_parts = {
|
|
".git", "__pycache__", "memories", "sessions", "state", "venv",
|
|
}
|
|
expected_roots = [
|
|
{"name": name, "exists": path.exists(), "is_symlink": path.is_symlink()}
|
|
for name, path in remote_scan_roots.items()
|
|
]
|
|
scanned_paths = set()
|
|
scanned_directories = set()
|
|
pending_paths = list(remote_scan_roots.values())
|
|
candidate_files = []
|
|
while pending_paths:
|
|
scan_path = pending_paths.pop()
|
|
if set(scan_path.parts) & excluded_profile_parts:
|
|
continue
|
|
try:
|
|
if scan_path.is_symlink():
|
|
remote_symlink_targets += 1
|
|
resolved = scan_path.resolve(strict=True)
|
|
if resolved.is_file():
|
|
candidate_files.append(resolved)
|
|
continue
|
|
if not resolved.is_dir():
|
|
continue
|
|
directory_identity = str(resolved)
|
|
if directory_identity in scanned_directories:
|
|
continue
|
|
scanned_directories.add(directory_identity)
|
|
pending_paths.extend(sorted(resolved.iterdir(), reverse=True))
|
|
except OSError as exc:
|
|
remote_scan_errors.append({
|
|
"path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(),
|
|
"error_type": type(exc).__name__,
|
|
})
|
|
for scan_path in sorted(candidate_files):
|
|
if set(scan_path.parts) & excluded_profile_parts:
|
|
continue
|
|
try:
|
|
path_identity = str(scan_path.resolve(strict=True))
|
|
if path_identity in scanned_paths:
|
|
continue
|
|
scanned_paths.add(path_identity)
|
|
scan_bytes = scan_path.read_bytes()
|
|
except OSError as exc:
|
|
remote_scan_errors.append({
|
|
"path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(),
|
|
"error_type": type(exc).__name__,
|
|
})
|
|
continue
|
|
remote_scanned_files += 1
|
|
remote_scanned_bytes += len(scan_bytes)
|
|
normalized = b" ".join(re.findall(rb"[a-z0-9_]+", scan_bytes.lower())).decode("ascii")
|
|
for prompt_id, marker in prompt_markers:
|
|
if marker in normalized:
|
|
remote_leakage_matches.append({
|
|
"prompt_id": prompt_id,
|
|
"marker_sha256": hashlib.sha256(marker.encode()).hexdigest(),
|
|
"path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(),
|
|
})
|
|
report["grounding_mode"] = OOS_GROUNDING_MODE
|
|
report["db_context_plugin_enabled"] = db_context_dir.exists()
|
|
report["read_only_tool_surface"] = tool_surface
|
|
report["readonly_guard_source_sha256"] = OOS_GUARD_SOURCE_SHA256
|
|
report["prompt_leakage_scan"] = OOS_PROMPT_LEAKAGE_SCAN
|
|
report["remote_temp_profile_prompt_leakage_scan"] = {
|
|
"scope": "full_model_visible_temp_profile_excluding_sessions_state_memories_and_venv",
|
|
"expected_roots": expected_roots,
|
|
"scanned_files": remote_scanned_files,
|
|
"scanned_bytes": remote_scanned_bytes,
|
|
"symlink_targets_encountered": remote_symlink_targets,
|
|
"errors": remote_scan_errors,
|
|
"matches": remote_leakage_matches,
|
|
"pass": bool(
|
|
all(item["exists"] for item in expected_roots)
|
|
and remote_scanned_files > 0
|
|
and remote_scanned_bytes > 0
|
|
and not remote_scan_errors
|
|
and not remote_leakage_matches
|
|
),
|
|
}
|
|
report["telegram_transport_deny"] = {
|
|
"enabled": set(patched_telegram_methods) == set(telegram_outbound_methods),
|
|
"expected_methods": list(telegram_outbound_methods),
|
|
"patched_methods": patched_telegram_methods,
|
|
"attempt_count": 0,
|
|
"attempts": [],
|
|
"runner_adapters_required_empty": True,
|
|
}
|
|
report["oos_max_turn_seconds"] = 180
|
|
report["preexecution_safety_gate"] = {
|
|
"status": "pass" if (
|
|
tool_surface.get("send_message_tool_enabled") is False
|
|
and tool_surface.get("mutating_bridge_commands_exposed") is False
|
|
and tool_surface.get("terminal_restricted_to_exact_wrapper") is True
|
|
and OOS_PROMPT_LEAKAGE_SCAN.get("pass") is True
|
|
and report["remote_temp_profile_prompt_leakage_scan"]["pass"] is True
|
|
and report["telegram_transport_deny"]["enabled"] is True
|
|
and report["telegram_transport_deny"]["attempt_count"] == 0
|
|
) else "fail",
|
|
"grounding_mode": OOS_GROUNDING_MODE,
|
|
}
|
|
write_report_snapshot(report)
|
|
if report["preexecution_safety_gate"]["status"] != "pass":
|
|
raise RuntimeError("OOS pre-execution safety gate failed")
|
|
|
|
source = SessionSource("""
|
|
script = script.replace(gateway_marker, injection)
|
|
turn_timeout_marker = "reply = await asyncio.wait_for(runner._handle_message(event), timeout=300)"
|
|
if script.count(turn_timeout_marker) != 1:
|
|
raise RuntimeError("remote harness per-turn timeout marker changed")
|
|
script = script.replace(
|
|
turn_timeout_marker,
|
|
"reply = await asyncio.wait_for(runner._handle_message(event), timeout=180)",
|
|
)
|
|
runner_marker = " runner = GatewayRunner()\n"
|
|
if script.count(runner_marker) != 1:
|
|
raise RuntimeError("remote harness runner marker changed")
|
|
script = script.replace(
|
|
runner_marker,
|
|
runner_marker
|
|
+ " runner.adapters.clear()\n"
|
|
+ " runner.delivery_router.adapters = runner.adapters\n"
|
|
+ " report['telegram_transport_deny']['runner_adapters_empty'] = not runner.adapters\n"
|
|
+ " write_report_snapshot(report)\n",
|
|
)
|
|
return script
|
|
|
|
|
|
def _remote_orphan_readback(parsed: dict[str, Any]) -> dict[str, Any]:
|
|
temp_profile = str((parsed.get("handler") or {}).get("temp_profile") or "")
|
|
if not temp_profile:
|
|
return {"status": "missing_temp_profile_identity", "no_matching_processes": False, "matches": []}
|
|
proc = subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-i",
|
|
str(handler.SSH_KEY),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
handler.VPS,
|
|
"ps -eo pid=,args=",
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=60,
|
|
)
|
|
matches = [handler.redact(line.strip()) for line in proc.stdout.splitlines() if temp_profile in line]
|
|
return {
|
|
"status": "ok" if proc.returncode == 0 else "ssh_error",
|
|
"returncode": proc.returncode,
|
|
"temp_profile_sha256": hashlib.sha256(temp_profile.encode()).hexdigest(),
|
|
"matches": matches,
|
|
"no_matching_processes": proc.returncode == 0 and not matches,
|
|
}
|
|
|
|
|
|
def _local_harness_git_state() -> dict[str, Any]:
|
|
head = subprocess.run(
|
|
["git", "rev-parse", "HEAD"],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
status = subprocess.run(
|
|
["git", "status", "--porcelain", "--untracked-files=all"],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
status_text = status.stdout if status.returncode == 0 else ""
|
|
status_lines = status_text.splitlines()
|
|
return {
|
|
"git_head": head.stdout.strip() if head.returncode == 0 else None,
|
|
"worktree_clean": status.returncode == 0 and not status_text.strip(),
|
|
"status_sha256": hashlib.sha256(status_text.encode()).hexdigest() if status.returncode == 0 else None,
|
|
"status_lines": status_lines,
|
|
"only_control_goal_untracked": status.returncode == 0 and status_lines == ["?? goal.md"],
|
|
}
|
|
|
|
|
|
def _portable_artifact_path(path: Path) -> str:
|
|
resolved = path.resolve()
|
|
try:
|
|
return str(resolved.relative_to(ROOT))
|
|
except ValueError:
|
|
return str(resolved)
|
|
|
|
|
|
def run_guarded_remote(
|
|
*,
|
|
prompts: list[dict[str, Any]],
|
|
suite_mode: str,
|
|
report_prefix: str,
|
|
prompt_note: str,
|
|
grounding_mode: str,
|
|
leakage_scan: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
"""Use the generic transport while substituting only the guarded builder."""
|
|
|
|
original_builder = handler.build_remote_script
|
|
|
|
def guarded_builder(run_id: str, **kwargs: Any) -> str:
|
|
return build_guarded_remote_script(
|
|
run_id,
|
|
prompts=kwargs.get("prompts") or prompts,
|
|
suite_mode=str(kwargs.get("suite_mode") or suite_mode),
|
|
report_prefix=str(kwargs.get("report_prefix") or report_prefix),
|
|
prompt_note=str(kwargs.get("prompt_note") or prompt_note),
|
|
grounding_mode=grounding_mode,
|
|
leakage_scan=leakage_scan,
|
|
)
|
|
|
|
handler.build_remote_script = guarded_builder
|
|
try:
|
|
remote = handler.run_remote(
|
|
prompts=prompts,
|
|
suite_mode=suite_mode,
|
|
report_prefix=report_prefix,
|
|
prompt_note=prompt_note,
|
|
)
|
|
finally:
|
|
handler.build_remote_script = original_builder
|
|
parsed = remote.get("parsed")
|
|
if isinstance(parsed, dict):
|
|
parsed["post_run_orphan_readback"] = _remote_orphan_readback(parsed)
|
|
return remote
|
|
|
|
|
|
def write_score_markdown(path: Path, report: dict[str, Any]) -> None:
|
|
score = report["score"]
|
|
lines = [
|
|
"# Working Leo m3taversal Out-of-Sample Score",
|
|
"",
|
|
f"Generated UTC: `{report['generated_at_utc']}`",
|
|
f"Pass: `{score['pass']}`",
|
|
f"Prompts: `{score['passes']}/{score['expected_prompt_count']}`",
|
|
f"DB counts changed: `{report['db_counts_changed']}`",
|
|
f"Live behavior manifest unchanged: `{report['live_behavior_manifest_unchanged']}`",
|
|
f"Service unchanged: `{report['service_unchanged']}`",
|
|
f"Temporary profile removed: `{report['temp_profile_removed']}`",
|
|
f"Database context injections: `{report['database_context_injection_count']}`",
|
|
f"Database context all OK: `{report['database_context_all_ok']}`",
|
|
f"Database response validations: `{report['database_response_validation_count']}`",
|
|
f"Database response validation all OK: `{report['database_response_validation_all_ok']}`",
|
|
f"Database-composed replacements: `{report['database_response_transform_count']}`",
|
|
f"Posted to Telegram: `{report['posted_to_telegram']}`",
|
|
"",
|
|
"## Prompt Scores",
|
|
"",
|
|
]
|
|
for item in score["scores"]:
|
|
lines.append(f"- `{item['prompt_id']}` / `{item['dimension']}`: `pass={item['pass']}`")
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Claim Ceiling",
|
|
"",
|
|
"A pass proves broad out-of-sample and same-session-memory behavior through the live VPS GatewayRunner "
|
|
"using a temporary profile, with no Telegram post and no DB count change. It does not prove human-visible "
|
|
"Telegram delivery, production DB apply, or GCP parity.",
|
|
"",
|
|
]
|
|
)
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def build_score_report(report: dict[str, Any], *, memory_token: str) -> dict[str, Any]:
|
|
score = benchmark.score_results(report.get("results") or [], memory_token=memory_token)
|
|
service_unchanged = bool((report.get("service_before_after") or {}).get("unchanged_from_preexisting_live_readback"))
|
|
return {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"mode": "working_leo_m3taversal_out_of_sample_live_vps_handler_score",
|
|
"source_results_json": str(RESULTS_JSON),
|
|
"memory_token": memory_token,
|
|
"score": score,
|
|
"db_counts_changed": report.get("db_counts_changed"),
|
|
"changed_live_profile": report.get("changed_live_profile"),
|
|
"live_behavior_manifest_unchanged": report.get("live_behavior_manifest_unchanged"),
|
|
"service_unchanged": service_unchanged,
|
|
"temp_profile_removed": report.get("temp_profile_removed"),
|
|
"database_context_injection_count": report.get("database_context_injection_count"),
|
|
"database_context_all_ok": report.get("database_context_all_ok"),
|
|
"database_response_validation_count": report.get("database_response_validation_count"),
|
|
"database_response_validation_all_ok": report.get("database_response_validation_all_ok"),
|
|
"database_response_transform_count": report.get("database_response_transform_count"),
|
|
"posted_to_telegram": report.get("posted_to_telegram"),
|
|
"production_db_apply_ran": False,
|
|
}
|
|
|
|
|
|
def score_report_passes(report: dict[str, Any], score_report: dict[str, Any]) -> bool:
|
|
return bool(
|
|
report.get("remote_returncode") == 0
|
|
and report.get("pass_runtime") is True
|
|
and report.get("db_counts_changed") is False
|
|
and report.get("changed_live_profile") is False
|
|
and report.get("live_behavior_manifest_unchanged") is True
|
|
and score_report["service_unchanged"]
|
|
and report.get("temp_profile_removed") is True
|
|
and report.get("database_context_all_ok") is True
|
|
and int(report.get("database_context_injection_count") or 0) >= len(report.get("results") or [])
|
|
and report.get("database_response_validation_all_ok") is True
|
|
and int(report.get("database_response_validation_count") or 0) >= len(report.get("results") or [])
|
|
and report.get("posted_to_telegram") is False
|
|
and score_report["score"]["pass"]
|
|
)
|
|
|
|
|
|
def write_protocol_score_markdown(path: Path, score: dict[str, Any]) -> None:
|
|
lines = [
|
|
"# Leo Blinded OOS Reasoning Trial",
|
|
"",
|
|
f"Generated UTC: `{score['generated_at_utc']}`",
|
|
f"Protocol: `{score['protocol_id']}` / `{score['protocol_hash_sha256']}`",
|
|
f"Trial: `{score['trial_id']}` / `{score['session_mode']}`",
|
|
f"Pass: `{score['pass']}`",
|
|
f"Grounded prompts: `{score['grounded_passes']}/{score['prompt_count']}`",
|
|
f"Grounded rate: `{score['grounded_pass_rate']:.3f}`",
|
|
f"No-DB ablation grounded rate: `{score['receipt_ablation']['grounded_pass_rate']:.3f}`",
|
|
"",
|
|
"## Prompt scores",
|
|
"",
|
|
]
|
|
for item in score["prompt_scores"]:
|
|
lines.append(
|
|
f"- `{item['prompt_id']}` / `{item['family_id']}`: semantic=`{item['semantic_pass']}`, "
|
|
f"subject=`{item['subject_alignment']}`, receipts=`{item['receipt_pass']}`, "
|
|
f"grounded=`{item['grounded_pass']}`"
|
|
)
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Safety",
|
|
"",
|
|
f"- Grounded live safety: `{score['top_level_safety']['pass']}`",
|
|
f"- Ablation live safety: `{score['baseline_top_level_safety']['pass']}`",
|
|
f"- Restart receipt: `{score['restart_receipt_validation']['pass']}`",
|
|
f"- Exact prompt binding: `{score['prompt_binding']['pass']}`",
|
|
"",
|
|
"## Claim ceiling",
|
|
"",
|
|
"This trial proves a direct live VPS GatewayRunner reply path with a fail-closed read-only tool surface, "
|
|
"full unchanged DB fingerprints, and no Telegram post. It does not prove Telegram-visible delivery or "
|
|
"any production database apply.",
|
|
"",
|
|
]
|
|
)
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def _run_protocol_mode(
|
|
protocol: dict[str, Any],
|
|
trial: dict[str, Any],
|
|
*,
|
|
grounding_mode: str,
|
|
output_dir: Path,
|
|
leakage_scan: dict[str, Any],
|
|
) -> tuple[dict[str, Any], Path]:
|
|
prompts = [
|
|
{"id": prompt["id"], "dimension": prompt["dimension"], "message": prompt["message"]}
|
|
for prompt in trial["prompts"]
|
|
]
|
|
safe_protocol_id = re.sub(r"[^A-Za-z0-9._-]", "-", str(protocol["protocol_id"]))
|
|
report_prefix = f"leo-oos-{safe_protocol_id}-{trial['trial_id']}-{grounding_mode}"
|
|
output_path = output_dir / f"{trial['trial_id']}-{grounding_mode}-handler.json"
|
|
remote = run_guarded_remote(
|
|
prompts=prompts,
|
|
suite_mode=f"live_vps_gatewayrunner_blinded_oos_{grounding_mode}",
|
|
report_prefix=report_prefix,
|
|
prompt_note=(
|
|
f"Frozen blinded protocol {protocol['protocol_hash_sha256']}; trial {trial['trial_id']}; "
|
|
f"grounding mode {grounding_mode}; direct handler only, no Telegram post, no database mutation."
|
|
),
|
|
grounding_mode=grounding_mode,
|
|
leakage_scan=leakage_scan,
|
|
)
|
|
report = handler.write_output(remote, output_json=output_path)
|
|
report["oos_harness_git_state"] = _local_harness_git_state()
|
|
report["protocol_id"] = protocol["protocol_id"]
|
|
report["protocol_hash_sha256"] = protocol["protocol_hash_sha256"]
|
|
report["trial_id"] = trial["trial_id"]
|
|
report["trial_prompt_set_sha256"] = trial["prompt_set_sha256"]
|
|
report["scorer_version"] = protocol["scorer_version"]
|
|
report["source_hashes"] = protocol["source_hashes"]
|
|
output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
return report, output_path
|
|
|
|
|
|
def _load_restart_receipt(path: Path | None, trial: dict[str, Any]) -> dict[str, Any] | None:
|
|
if trial["session_mode"] != "post_restart_clean_session":
|
|
return None
|
|
if path is None:
|
|
raise SystemExit("--restart-receipt is required for a post-restart trial")
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def _ssh_readback(command: str, *, timeout: int = 120) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-i",
|
|
str(handler.SSH_KEY),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
handler.VPS,
|
|
command,
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
)
|
|
|
|
|
|
def _deploy_identity() -> dict[str, Any]:
|
|
proc = _ssh_readback(
|
|
"cd /opt/teleo-eval/workspaces/deploy-infra && "
|
|
"printf 'head=' && git rev-parse HEAD && "
|
|
"printf 'stamp=' && cat /opt/teleo-eval/.last-deploy-sha",
|
|
timeout=60,
|
|
)
|
|
values: dict[str, str] = {}
|
|
for line in proc.stdout.splitlines():
|
|
if "=" in line:
|
|
key, value = line.split("=", 1)
|
|
values[key] = value.strip()
|
|
return {
|
|
"returncode": proc.returncode,
|
|
"head": values.get("head"),
|
|
"stamp": values.get("stamp"),
|
|
"stderr": handler.redact(proc.stderr),
|
|
}
|
|
|
|
|
|
def _restart_state_probe(
|
|
protocol: dict[str, Any],
|
|
*,
|
|
label: str,
|
|
output_dir: Path,
|
|
leakage_scan: dict[str, Any],
|
|
) -> tuple[dict[str, Any], Path]:
|
|
output_path = output_dir / f"restart-{label}-state.json"
|
|
remote = run_guarded_remote(
|
|
prompts=[],
|
|
suite_mode=f"live_vps_gateway_restart_{label}_readonly_state_probe",
|
|
report_prefix=f"leo-oos-restart-{label}-{protocol['protocol_id']}",
|
|
prompt_note=(
|
|
f"Read-only restart {label} state probe for frozen protocol {protocol['protocol_hash_sha256']}; "
|
|
"zero prompts, no Telegram post, no database mutation."
|
|
),
|
|
grounding_mode="grounded",
|
|
leakage_scan=leakage_scan,
|
|
)
|
|
report = handler.write_output(remote, output_json=output_path)
|
|
report["oos_harness_git_state"] = _local_harness_git_state()
|
|
report["protocol_id"] = protocol["protocol_id"]
|
|
report["protocol_hash_sha256"] = protocol["protocol_hash_sha256"]
|
|
report["source_hashes"] = protocol["source_hashes"]
|
|
output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
return report, output_path
|
|
|
|
|
|
def _restart_probe_passes(report: dict[str, Any]) -> bool:
|
|
before = report.get("db_fingerprint_before") or {}
|
|
after = report.get("db_fingerprint_after") or {}
|
|
counts_before = report.get("db_counts_before")
|
|
counts_after = report.get("db_counts_after")
|
|
transport = report.get("telegram_transport_deny") or {}
|
|
return bool(
|
|
report.get("remote_returncode") == 0
|
|
and report.get("pass_runtime") is True
|
|
and report.get("posted_to_telegram") is False
|
|
and report.get("db_counts_changed") is False
|
|
and isinstance(counts_before, dict)
|
|
and bool(counts_before)
|
|
and counts_before == counts_after
|
|
and all(isinstance(value, int) and not isinstance(value, bool) for value in counts_before.values())
|
|
and report.get("db_fingerprint_unchanged") is True
|
|
and before.get("status") == "ok"
|
|
and after.get("status") == "ok"
|
|
and bool(re.fullmatch(r"[0-9a-f]{64}", str(before.get("fingerprint_sha256") or "")))
|
|
and before.get("fingerprint_sha256") == after.get("fingerprint_sha256")
|
|
and (report.get("preexecution_safety_gate") or {}).get("status") == "pass"
|
|
and transport.get("enabled") is True
|
|
and isinstance(transport.get("attempt_count"), int)
|
|
and not isinstance(transport.get("attempt_count"), bool)
|
|
and transport.get("attempt_count") == 0
|
|
and transport.get("runner_adapters_empty") is True
|
|
and report.get("temp_profile_removed") is True
|
|
and (report.get("post_run_orphan_readback") or {}).get("no_matching_processes") is True
|
|
)
|
|
|
|
|
|
def collect_restart_receipt(args: argparse.Namespace) -> int:
|
|
protocol = json.loads(args.protocol.read_text(encoding="utf-8"))
|
|
validation = protocol_lib.validate_protocol(protocol, verify_source_hashes=True)
|
|
if not validation["pass"]:
|
|
raise SystemExit(f"frozen protocol validation failed: {validation['issues']}")
|
|
restart_trials = [
|
|
item for item in protocol.get("trials") or [] if item.get("session_mode") == "post_restart_clean_session"
|
|
]
|
|
if len(restart_trials) != 1:
|
|
raise SystemExit("frozen protocol must identify exactly one post-restart trial")
|
|
next_trial = restart_trials[0]
|
|
leakage_scan = prompt_leakage_scan(protocol)
|
|
if not leakage_scan["pass"]:
|
|
raise SystemExit(f"runtime prompt leakage detected: {leakage_scan['exact_prompt_matches']}")
|
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
before_deploy = _deploy_identity()
|
|
before_report, before_path = _restart_state_probe(
|
|
protocol,
|
|
label="before",
|
|
output_dir=args.output_dir,
|
|
leakage_scan=leakage_scan,
|
|
)
|
|
if (
|
|
before_deploy.get("returncode") != 0
|
|
or not re.fullmatch(r"[0-9a-f]{40}", str(before_deploy.get("head") or ""))
|
|
or before_deploy.get("head") != before_deploy.get("stamp")
|
|
):
|
|
raise SystemExit(f"deploy identity preflight failed: {before_deploy}")
|
|
if not _restart_probe_passes(before_report):
|
|
raise SystemExit(f"read-only pre-restart probe failed: {before_path}")
|
|
|
|
restart_started_at_utc = datetime.now(timezone.utc).isoformat()
|
|
restart = _ssh_readback(
|
|
"systemctl restart leoclean-gateway.service && "
|
|
"for i in $(seq 1 60); do "
|
|
"if [ \"$(systemctl is-active leoclean-gateway.service)\" = active ]; then break; fi; sleep 1; done; "
|
|
"systemctl show leoclean-gateway.service "
|
|
"-p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp",
|
|
timeout=120,
|
|
)
|
|
restart_ended_at_utc = datetime.now(timezone.utc).isoformat()
|
|
after_deploy = _deploy_identity()
|
|
after_report, after_path = _restart_state_probe(
|
|
protocol,
|
|
label="after",
|
|
output_dir=args.output_dir,
|
|
leakage_scan=leakage_scan,
|
|
)
|
|
service_before = (before_report.get("service_before_after") or {}).get("after") or {}
|
|
service_after = after_report.get("before_service") or {}
|
|
fingerprint_before = before_report.get("db_fingerprint_after") or {}
|
|
fingerprint_after = after_report.get("db_fingerprint_before") or {}
|
|
counts_before = before_report.get("db_counts_after") or {}
|
|
counts_after = after_report.get("db_counts_before") or {}
|
|
receipt = {
|
|
"schema": "livingip.leoGatewayRestartReceipt.v1",
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"protocol_id": protocol["protocol_id"],
|
|
"protocol_hash_sha256": protocol["protocol_hash_sha256"],
|
|
"next_trial_id": next_trial["trial_id"],
|
|
"next_trial_prompt_set_sha256": next_trial["prompt_set_sha256"],
|
|
"authorization_basis": "goal.md requires restart trials; service restart only, no Telegram post and no DB mutation",
|
|
"restart_command": "systemctl restart leoclean-gateway.service",
|
|
"restart_started_at_utc": restart_started_at_utc,
|
|
"restart_ended_at_utc": restart_ended_at_utc,
|
|
"restart_returncode": restart.returncode,
|
|
"restart_stdout": handler.redact(restart.stdout),
|
|
"restart_stderr": handler.redact(restart.stderr),
|
|
"service_before": service_before,
|
|
"service_after": service_after,
|
|
"deploy_before": before_deploy,
|
|
"deploy_after": after_deploy,
|
|
"db_counts_before": counts_before,
|
|
"db_counts_after": counts_after,
|
|
"db_counts_changed": counts_before != counts_after,
|
|
"db_fingerprint_before": fingerprint_before,
|
|
"db_fingerprint_after": fingerprint_after,
|
|
"db_fingerprint_unchanged": bool(
|
|
fingerprint_before.get("status") == "ok"
|
|
and fingerprint_after.get("status") == "ok"
|
|
and fingerprint_before.get("fingerprint_sha256")
|
|
== fingerprint_after.get("fingerprint_sha256")
|
|
),
|
|
"posted_to_telegram": False,
|
|
"before_probe": {
|
|
"path": _portable_artifact_path(before_path),
|
|
"sha256": hashlib.sha256(before_path.read_bytes()).hexdigest(),
|
|
"pass": _restart_probe_passes(before_report),
|
|
},
|
|
"after_probe": {
|
|
"path": _portable_artifact_path(after_path),
|
|
"sha256": hashlib.sha256(after_path.read_bytes()).hexdigest(),
|
|
"pass": _restart_probe_passes(after_report),
|
|
},
|
|
}
|
|
receipt["checks"] = {
|
|
"restart_command_succeeded": receipt["restart_returncode"] == 0,
|
|
"service_active_after": service_after.get("ActiveState") == "active"
|
|
and service_after.get("SubState") == "running",
|
|
"service_pid_changed": bool(
|
|
service_before.get("MainPID") and service_before.get("MainPID") != service_after.get("MainPID")
|
|
),
|
|
"deploy_identity_unchanged": before_deploy.get("returncode") == 0
|
|
and after_deploy.get("returncode") == 0
|
|
and bool(re.fullmatch(r"[0-9a-f]{40}", str(before_deploy.get("head") or "")))
|
|
and before_deploy.get("head")
|
|
== before_deploy.get("stamp")
|
|
== after_deploy.get("head")
|
|
== after_deploy.get("stamp"),
|
|
"db_counts_unchanged": receipt["db_counts_changed"] is False,
|
|
"db_counts_complete": isinstance(counts_before, dict)
|
|
and bool(counts_before)
|
|
and counts_before == counts_after
|
|
and all(isinstance(value, int) and not isinstance(value, bool) for value in counts_before.values()),
|
|
"db_fingerprint_unchanged": receipt["db_fingerprint_unchanged"] is True,
|
|
"db_fingerprint_complete": bool(
|
|
re.fullmatch(r"[0-9a-f]{64}", str(fingerprint_before.get("fingerprint_sha256") or ""))
|
|
and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256")
|
|
),
|
|
"before_probe_passed": receipt["before_probe"]["pass"] is True,
|
|
"after_probe_passed": receipt["after_probe"]["pass"] is True,
|
|
}
|
|
receipt["pass"] = all(receipt["checks"].values())
|
|
args.collect_restart_receipt.parent.mkdir(parents=True, exist_ok=True)
|
|
args.collect_restart_receipt.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"restart_receipt": str(args.collect_restart_receipt),
|
|
"pass": receipt["pass"],
|
|
"checks": receipt["checks"],
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0 if receipt["pass"] else 1
|
|
|
|
|
|
def run_or_score_protocol_trial(args: argparse.Namespace) -> int:
|
|
protocol = json.loads(args.protocol.read_text(encoding="utf-8"))
|
|
validation = protocol_lib.validate_protocol(protocol, verify_source_hashes=True)
|
|
if not validation["pass"]:
|
|
raise SystemExit(f"frozen protocol validation failed: {validation['issues']}")
|
|
trial = next((item for item in protocol["trials"] if item["trial_id"] == args.trial_id), None)
|
|
if trial is None:
|
|
raise SystemExit(f"unknown trial id: {args.trial_id}")
|
|
leakage_scan = prompt_leakage_scan(protocol)
|
|
if not leakage_scan["pass"]:
|
|
raise SystemExit(f"runtime prompt leakage detected: {leakage_scan['exact_prompt_matches']}")
|
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
restart_receipt = _load_restart_receipt(args.restart_receipt, trial)
|
|
|
|
if bool(args.grounded_report) != bool(args.baseline_report):
|
|
raise SystemExit("--grounded-report and --baseline-report must be supplied together")
|
|
if args.grounded_report:
|
|
grounded_report = json.loads(args.grounded_report.read_text(encoding="utf-8"))
|
|
baseline_report = json.loads(args.baseline_report.read_text(encoding="utf-8"))
|
|
grounded_path = args.grounded_report
|
|
baseline_path = args.baseline_report
|
|
elif args.grounding_mode:
|
|
report, output_path = _run_protocol_mode(
|
|
protocol,
|
|
trial,
|
|
grounding_mode=args.grounding_mode,
|
|
output_dir=args.output_dir,
|
|
leakage_scan=leakage_scan,
|
|
)
|
|
mode_safety = protocol_lib._top_level_safety(
|
|
report,
|
|
require_handler_safety_gate=args.grounding_mode == "grounded",
|
|
)
|
|
mode_checks = {
|
|
"expected_grounding_mode": report.get("grounding_mode") == args.grounding_mode,
|
|
"db_context_state": report.get("db_context_plugin_enabled")
|
|
is (args.grounding_mode == "grounded"),
|
|
"tool_surface_mode": (report.get("read_only_tool_surface") or {}).get("mode")
|
|
== ("read_only_kb" if args.grounding_mode == "grounded" else "no_db_ablation"),
|
|
"zero_context_in_ablation": args.grounding_mode != "db_tool_ablated"
|
|
or all(not (item.get("database_context_trace") or []) for item in report.get("results") or []),
|
|
}
|
|
mode_pass = mode_safety["pass"] and all(mode_checks.values())
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"report": str(output_path),
|
|
"grounding_mode": args.grounding_mode,
|
|
"handler_safety_gate": report.get("safety_gate"),
|
|
"post_run_orphan_readback": report.get("post_run_orphan_readback"),
|
|
"mode_safety": mode_safety,
|
|
"mode_checks": mode_checks,
|
|
"pass": mode_pass,
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0 if mode_pass else 1
|
|
else:
|
|
grounded_report, grounded_path = _run_protocol_mode(
|
|
protocol,
|
|
trial,
|
|
grounding_mode="grounded",
|
|
output_dir=args.output_dir,
|
|
leakage_scan=leakage_scan,
|
|
)
|
|
baseline_report, baseline_path = _run_protocol_mode(
|
|
protocol,
|
|
trial,
|
|
grounding_mode="db_tool_ablated",
|
|
output_dir=args.output_dir,
|
|
leakage_scan=leakage_scan,
|
|
)
|
|
|
|
score = protocol_lib.score_live_trial(
|
|
protocol,
|
|
trial["trial_id"],
|
|
grounded_report,
|
|
baseline_report=baseline_report,
|
|
restart_receipt=restart_receipt,
|
|
)
|
|
score["grounded_report_path"] = _portable_artifact_path(grounded_path)
|
|
score["baseline_report_path"] = _portable_artifact_path(baseline_path)
|
|
score["grounded_report_sha256"] = hashlib.sha256(grounded_path.read_bytes()).hexdigest()
|
|
score["baseline_report_sha256"] = hashlib.sha256(baseline_path.read_bytes()).hexdigest()
|
|
if restart_receipt is not None and args.restart_receipt is not None:
|
|
score["restart_receipt_path"] = _portable_artifact_path(args.restart_receipt)
|
|
score["restart_receipt_sha256"] = hashlib.sha256(args.restart_receipt.read_bytes()).hexdigest()
|
|
score["restart_receipt_payload_sha256"] = protocol_lib.canonical_sha256(restart_receipt)
|
|
score_path = args.output_dir / f"{trial['trial_id']}-score.json"
|
|
markdown_path = args.output_dir / f"{trial['trial_id']}-score.md"
|
|
score_path.write_text(json.dumps(score, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
write_protocol_score_markdown(markdown_path, score)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"grounded_report": str(grounded_path),
|
|
"baseline_report": str(baseline_path),
|
|
"score_json": str(score_path),
|
|
"score_markdown": str(markdown_path),
|
|
"pass": score["pass"],
|
|
"grounded_rate": score["grounded_pass_rate"],
|
|
"baseline_grounded_rate": score["receipt_ablation"]["grounded_pass_rate"],
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0 if score["pass"] else 1
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--score-existing",
|
|
action="store_true",
|
|
help="Rescore the retained live transcript without making another remote model call.",
|
|
)
|
|
parser.add_argument("--protocol", type=Path)
|
|
parser.add_argument("--trial-id")
|
|
parser.add_argument("--output-dir", type=Path, default=PROTOCOL_REPORT_DIR)
|
|
parser.add_argument("--grounding-mode", choices=GROUNDING_MODES)
|
|
parser.add_argument("--grounded-report", type=Path)
|
|
parser.add_argument("--baseline-report", type=Path)
|
|
parser.add_argument("--restart-receipt", type=Path)
|
|
parser.add_argument("--collect-restart-receipt", type=Path)
|
|
args = parser.parse_args()
|
|
|
|
if args.collect_restart_receipt:
|
|
if not args.protocol or args.trial_id:
|
|
raise SystemExit("--collect-restart-receipt requires --protocol and does not accept --trial-id")
|
|
return collect_restart_receipt(args)
|
|
if args.protocol or args.trial_id:
|
|
if not args.protocol or not args.trial_id:
|
|
raise SystemExit("--protocol and --trial-id are required together")
|
|
return run_or_score_protocol_trial(args)
|
|
if not args.score_existing:
|
|
raise SystemExit(
|
|
"refusing the legacy fixed live suite; use --protocol and --trial-id for a frozen guarded trial"
|
|
)
|
|
|
|
if args.score_existing:
|
|
report = json.loads(RESULTS_JSON.read_text(encoding="utf-8"))
|
|
previous_score = json.loads(SCORE_JSON.read_text(encoding="utf-8"))
|
|
memory_token = str(previous_score["memory_token"])
|
|
|
|
score_report = build_score_report(report, memory_token=memory_token)
|
|
SCORE_JSON.write_text(json.dumps(score_report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
write_score_markdown(SCORE_MARKDOWN, score_report)
|
|
safe_pass = score_report_passes(report, score_report)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"results_json": str(RESULTS_JSON),
|
|
"score_json": str(SCORE_JSON),
|
|
"score_markdown": str(SCORE_MARKDOWN),
|
|
"pass": safe_pass,
|
|
"prompts": f"{score_report['score']['passes']}/{score_report['score']['expected_prompt_count']}",
|
|
"rescored_existing": args.score_existing,
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0 if safe_pass else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|