teleo-infrastructure/scripts/run_leo_clone_bound_handler_checkpoint.py

2374 lines
97 KiB
Python

#!/usr/bin/env python3
"""Run one no-send Leo GatewayRunner checkpoint against a disposable Postgres target.
Run this command on the Leo VPS as the ``teleo`` user while the disposable
container/database exists. The command copies the live profile, detaches and
patches only the temporary Teleo KB bridge, invokes the real GatewayRunner
handler, and removes the temporary profile and session before returning.
"""
from __future__ import annotations
import argparse
import asyncio
import hashlib
import inspect
import json
import multiprocessing
import os
import re
import shlex
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import traceback
import uuid
from collections.abc import Iterator
from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent")
PRODUCTION_CONTAINER = "teleo-pg"
PRODUCTION_DB = "teleo"
GATEWAY_SERVICE = "leoclean-gateway.service"
DISPOSABLE_LABEL_KEY = "com.livingip.leo.checkpoint"
DISPOSABLE_LABEL_VALUE = "disposable"
SYSTEM_EXEC_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
SYSTEM_DOCKER = "/usr/bin/docker"
DEFAULT_CHAT_ID = "-5146042086"
DEFAULT_USER_ID = "9070919"
DEFAULT_USER_NAME = "codex clone checkpoint"
EXPECTED_STATES = ("pending_review", "approved", "applied")
READ_ONLY_BRIDGE_COMMANDS = frozenset(
{
"context",
"search",
"show",
"evidence",
"edges",
"list-proposals",
"search-proposals",
"show-proposal",
"decision-matrix-status",
}
)
HARMLESS_TRAILING_REDIRECT_TOKENS = frozenset({"2>&1"})
CANONICAL_TABLES = (
"public.claims",
"public.sources",
"public.claim_evidence",
"public.claim_edges",
"public.reasoning_tools",
)
COUNT_TABLES = (*CANONICAL_TABLES, "kb_stage.kb_proposals")
UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b")
SAFE_DOCKER_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*\Z")
SAFE_DB_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_$.-]*\Z")
SAFE_ENV_NAME_RE = re.compile(r"[A-Z][A-Z0-9_]*\Z")
SECRET_PATTERNS = (
re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{20,}\b"),
re.compile(r"\bsk-(?:or-v1-)?[A-Za-z0-9_-]{16,}\b"),
re.compile(r"\beyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{10,}(?:\.[A-Za-z0-9_-]{10,})?\b"),
re.compile(r"(Authorization:\s*Bearer\s+)([A-Za-z0-9._-]+)", re.IGNORECASE),
re.compile(r"((?:api[_-]?key|token|secret|password)\s*[=:]\s*)([^\s,;]+)", re.IGNORECASE),
re.compile(
r"((?:access_token|refresh_token|api_key|authorization|cookie)[\"']?\s*:\s*[\"'])([^\"']+)",
re.IGNORECASE,
),
)
SENSITIVE_KEY_RE = re.compile(
r"(?:^|_)(?:access_token|refresh_token|api_key|authorization|bot_token|client_secret|cookie|credential|key|"
r"password|private_key|secret|signing_key|token)(?:$|_)",
re.IGNORECASE,
)
_ACTIVE_TEMP_ROOTS: set[Path] = set()
_ACTIVE_GATEWAY_CHILDREN: set[Any] = set()
_LAST_TERMINATION_SIGNAL: int | None = None
class CheckpointError(RuntimeError):
"""Raised when a checkpoint cannot produce trustworthy proof."""
class TerminationRequested(BaseException):
"""Turn an interruptible termination signal into normal ``finally`` cleanup."""
def __init__(self, signum: int) -> None:
super().__init__(f"termination signal {signum}")
self.signum = signum
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def sha256_bytes(value: bytes) -> str:
return hashlib.sha256(value).hexdigest()
def json_sha256(value: Any) -> str:
encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()
return sha256_bytes(encoded)
def redact_text(value: str, *, secret_values: tuple[str, ...] = ()) -> str:
result = value
for secret in sorted({item for item in secret_values if len(item) >= 4}, key=len, reverse=True):
result = result.replace(secret, "[REDACTED_SECRET]")
for pattern in SECRET_PATTERNS:
if pattern.groups >= 2:
result = pattern.sub(r"\1[REDACTED]", result)
else:
result = pattern.sub("[REDACTED_TOKEN]", result)
return result
def redact_value(value: Any, *, key: str | None = None, secret_values: tuple[str, ...] = ()) -> Any:
if key and SENSITIVE_KEY_RE.search(key) and isinstance(value, (str, bytes, dict, list, tuple)):
return "[REDACTED]"
if isinstance(value, dict):
return {
item_key: redact_value(item, key=str(item_key), secret_values=secret_values)
for item_key, item in value.items()
}
if isinstance(value, list):
return [redact_value(item, secret_values=secret_values) for item in value]
if isinstance(value, tuple):
return [redact_value(item, secret_values=secret_values) for item in value]
if isinstance(value, str):
return redact_text(value, secret_values=secret_values)
return value
def run_command(command: list[str], *, input_text: str | None = None) -> subprocess.CompletedProcess[str]:
return subprocess.run(
command,
input=input_text,
text=True,
capture_output=True,
check=False,
)
def _command_failure(command: list[str], proc: subprocess.CompletedProcess[str]) -> CheckpointError:
detail = redact_text((proc.stderr or proc.stdout).strip())
return CheckpointError(f"command failed ({proc.returncode}): {shlex.join(command)}: {detail}")
def service_state() -> dict[str, Any]:
command = [
"systemctl",
"show",
GATEWAY_SERVICE,
"-p",
"ActiveState",
"-p",
"SubState",
"-p",
"MainPID",
"-p",
"NRestarts",
"-p",
"ExecMainStartTimestamp",
"-p",
"User",
"-p",
"WorkingDirectory",
"--no-pager",
]
proc = run_command(command)
state: dict[str, Any] = {"returncode": proc.returncode}
for line in proc.stdout.splitlines():
key, separator, value = line.partition("=")
if separator:
state[key] = value
if proc.returncode != 0:
state["error"] = (proc.stderr or proc.stdout).strip()
return state
def container_identity(container: str) -> dict[str, Any]:
command = ["docker", "inspect", container]
proc = run_command(command)
if proc.returncode != 0:
raise _command_failure(command, proc)
try:
payload = json.loads(proc.stdout)
item = payload[0]
except (json.JSONDecodeError, IndexError, KeyError, TypeError) as exc:
raise CheckpointError(f"docker inspect returned invalid JSON for {container!r}") from exc
identity = str(item.get("Id") or "")
if not identity:
raise CheckpointError(f"docker inspect returned an invalid identity for {container!r}")
config = item.get("Config") if isinstance(item.get("Config"), dict) else {}
host_config = item.get("HostConfig") if isinstance(item.get("HostConfig"), dict) else {}
network = item.get("NetworkSettings") if isinstance(item.get("NetworkSettings"), dict) else {}
mounts = item.get("Mounts") if isinstance(item.get("Mounts"), list) else []
return {
"id": identity,
"name": str(item.get("Name") or "").lstrip("/"),
"image": str(config.get("Image") or ""),
"labels": dict(config.get("Labels") or {}),
"network_mode": host_config.get("NetworkMode"),
"ports": network.get("Ports") or {},
"mount_sources": sorted(
str(mount.get("Source")) for mount in mounts if isinstance(mount, dict) and mount.get("Source")
),
"started_at": (item.get("State") or {}).get("StartedAt") if isinstance(item.get("State"), dict) else None,
}
def validate_disposable_target_identity(target: dict[str, Any], production: dict[str, Any]) -> dict[str, bool]:
target_mounts = set(target.get("mount_sources") or [])
production_mounts = set(production.get("mount_sources") or [])
checks = {
"different_container_id": bool(target.get("id")) and target.get("id") != production.get("id"),
"disposable_label_present": (target.get("labels") or {}).get(DISPOSABLE_LABEL_KEY) == DISPOSABLE_LABEL_VALUE,
"network_disabled": target.get("network_mode") == "none",
"no_published_ports": not bool(target.get("ports")),
"no_shared_mount_sources": not bool(target_mounts & production_mounts),
}
if not all(checks.values()):
failed = sorted(name for name, value in checks.items() if not value)
raise CheckpointError(f"supplied target is not a proven disposable isolated container: {failed}")
return checks
def _psql_json(container: str, database: str, sql: str) -> Any:
command = [
"docker",
"exec",
"-i",
container,
"psql",
"-U",
"postgres",
"-d",
database,
"-At",
"-q",
"-v",
"ON_ERROR_STOP=1",
]
proc = run_command(command, input_text=sql)
if proc.returncode != 0:
raise _command_failure(command, proc)
raw = proc.stdout.strip()
if not raw:
raise CheckpointError(f"psql returned no JSON for {container}/{database}")
try:
return json.loads(raw.splitlines()[-1])
except json.JSONDecodeError as exc:
raise CheckpointError(f"psql returned invalid JSON for {container}/{database}: {exc}") from exc
def _sql_identifier(value: str) -> str:
return '"' + value.replace('"', '""') + '"'
def target_read_only_setting(container: str, database: str) -> dict[str, Any]:
return _psql_json(
container,
database,
r"""\set ON_ERROR_STOP on
select jsonb_build_object(
'already_forced_read_only', exists (
select 1
from pg_db_role_setting settings
join pg_roles role_row on role_row.oid = settings.setrole
join pg_database database_row on database_row.oid = settings.setdatabase
where role_row.rolname = 'postgres'
and database_row.datname = current_database()
and 'default_transaction_read_only=on' = any(settings.setconfig)
)
)::text;
""",
)
def configure_target_read_only(container: str, database: str, *, enabled: bool) -> dict[str, Any]:
database_identifier = _sql_identifier(database)
if enabled:
setting = target_read_only_setting(container, database)
command_sql = f"alter role postgres in database {database_identifier} set default_transaction_read_only = on;"
else:
setting = {"already_forced_read_only": False}
command_sql = (
"set default_transaction_read_only = off; "
f"alter role postgres in database {database_identifier} reset default_transaction_read_only;"
)
command = [
"docker",
"exec",
"-i",
container,
"psql",
"-U",
"postgres",
"-d",
database,
"-At",
"-q",
"-v",
"ON_ERROR_STOP=1",
]
proc = run_command(command, input_text=command_sql)
if proc.returncode != 0:
raise _command_failure(command, proc)
verification = _psql_json(
container,
database,
"select jsonb_build_object('transaction_read_only', current_setting('transaction_read_only'))::text;",
)
expected = "on" if enabled else "off"
if verification.get("transaction_read_only") != expected:
raise CheckpointError(f"target read-only verification expected {expected!r}: {verification}")
return {
"enabled": enabled,
"already_forced_read_only": bool(setting.get("already_forced_read_only")),
"verified_transaction_read_only": verification.get("transaction_read_only"),
}
def database_counts(
container: str,
database: str,
*,
tables: tuple[str, ...] = COUNT_TABLES,
) -> dict[str, int]:
pairs = ",\n ".join(f"'{table}', (select count(*) from {table})" for table in tables)
result = _psql_json(
container,
database,
f"""\\set ON_ERROR_STOP on
begin transaction read only;
select jsonb_build_object(
{pairs}
)::text;
rollback;
""",
)
return {str(key): int(value) for key, value in result.items()}
def database_guard_snapshot(
container: str,
database: str,
*,
tables: tuple[str, ...] = COUNT_TABLES,
) -> dict[str, Any]:
count_pairs = ",\n ".join(f"'{table}', (select count(*) from {table})" for table in tables)
hash_pairs = ",\n ".join(
f"'{table}', (select md5(coalesce(string_agg(to_jsonb(t)::text, E'\\n' "
f"order by to_jsonb(t)::text), '')) from {table} t)"
for table in tables
)
result = _psql_json(
container,
database,
rf"""\set ON_ERROR_STOP on
begin transaction read only;
select jsonb_build_object(
'database_identity', jsonb_build_object(
'current_database', current_database(),
'system_identifier', (select system_identifier::text from pg_control_system()),
'postmaster_started_at', pg_postmaster_start_time()::text,
'transaction_read_only', current_setting('transaction_read_only')
),
'counts', jsonb_build_object(
{count_pairs}
),
'rowset_md5', jsonb_build_object(
{hash_pairs}
)
)::text;
rollback;
""",
)
return {
"database_identity": {str(key): value for key, value in result["database_identity"].items()},
"counts": {str(key): int(value) for key, value in result["counts"].items()},
"rowset_md5": {str(key): str(value) for key, value in result["rowset_md5"].items()},
}
def target_checkpoint(container: str, database: str) -> dict[str, Any]:
pairs = ",\n ".join(f"'{table}', (select count(*) from {table})" for table in COUNT_TABLES)
checkpoint = _psql_json(
container,
database,
f"""\\set ON_ERROR_STOP on
begin transaction read only;
select jsonb_build_object(
'database_identity', jsonb_build_object(
'current_database', current_database(),
'database_oid', (select oid::text from pg_database where datname = current_database()),
'system_identifier', (select system_identifier::text from pg_control_system()),
'server_address', coalesce(inet_server_addr()::text, 'local_socket'),
'server_port', inet_server_port(),
'postmaster_started_at', pg_postmaster_start_time()::text,
'transaction_read_only', current_setting('transaction_read_only'),
'transaction_snapshot', txid_current_snapshot()::text
),
'counts', jsonb_build_object(
{pairs}
),
'proposals', coalesce(
(select jsonb_agg(to_jsonb(p) order by p.id::text) from kb_stage.kb_proposals p),
'[]'::jsonb
)
)::text;
rollback;
""",
)
checkpoint["sha256"] = json_sha256(checkpoint)
return checkpoint
def proposal_receipt_view(proposal: dict[str, Any]) -> dict[str, Any]:
payload = proposal.get("payload")
return {
"id": proposal.get("id"),
"proposal_type": proposal.get("proposal_type"),
"status": proposal.get("status"),
"proposed_by": proposal.get("proposed_by"),
"channel": proposal.get("channel"),
"source_ref": proposal.get("source_ref"),
"created_at": proposal.get("created_at"),
"reviewed_at": proposal.get("reviewed_at"),
"reviewed_by_handle": proposal.get("reviewed_by_handle"),
"applied_at": proposal.get("applied_at"),
"applied_by_handle": proposal.get("applied_by_handle"),
"payload_sha256": json_sha256(payload),
"payload_uuid_values": sorted(_uuid_values(payload)),
}
def target_checkpoint_receipt_view(checkpoint: dict[str, Any]) -> dict[str, Any]:
return {
"database_identity": checkpoint.get("database_identity"),
"counts": checkpoint.get("counts"),
"proposals": [proposal_receipt_view(item) for item in checkpoint.get("proposals", [])],
"sha256": checkpoint.get("sha256"),
}
def canonical_rows_receipt_view(rows_by_table: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
identity_fields = {
"id",
"claim_id",
"source_id",
"from_claim",
"to_claim",
"reasoning_tool_id",
"slug",
"key",
"type",
"edge_type",
}
result: dict[str, list[dict[str, Any]]] = {}
for table, rows in rows_by_table.items():
result[table] = [
{
**{key: value for key, value in row.items() if key in identity_fields},
"row_sha256": json_sha256(row),
}
for row in rows
if isinstance(row, dict)
]
return result
def _sql_text(value: str) -> str:
return "'" + value.replace("'", "''") + "'"
def related_canonical_rows(container: str, database: str, identifiers: set[str]) -> dict[str, Any]:
empty = {table: [] for table in CANONICAL_TABLES}
if not identifiers:
return empty
values = ", ".join(_sql_text(value) for value in sorted(identifiers))
pairs = []
for table in CANONICAL_TABLES:
pairs.append(
f"""'{table}', coalesce((
select jsonb_agg(to_jsonb(row_data) order by to_jsonb(row_data)::text)
from {table} row_data
where exists (
select 1
from jsonb_each_text(to_jsonb(row_data)) item
where item.value = any(array[{values}]::text[])
)
), '[]'::jsonb)"""
)
return _psql_json(
container,
database,
"""\\set ON_ERROR_STOP on
begin transaction read only;
select jsonb_build_object(
"""
+ ",\n ".join(pairs)
+ """
)::text;
rollback;
""",
)
class ProfileCopyPolicy:
"""Exclude runtime state and credentials without reading their contents."""
OMITTED_DIRECTORIES = frozenset(
{
".claude",
".codex",
".git",
".pytest_cache",
"__pycache__",
"auth",
"backups",
"cache",
"credentials",
"image_cache",
"logs",
"memory_backups",
"secrets",
"sessions",
"state",
}
)
OMITTED_EXACT_FILES = frozenset(
{
".env",
"auth.json",
"auth.lock",
"credentials.json",
"gateway.pid",
}
)
STATE_SUFFIXES = (".db", ".db-shm", ".db-wal", ".sqlite", ".sqlite3", ".sqlite-shm", ".sqlite-wal")
SECRET_SUFFIXES = (".key", ".pem", ".p12", ".pfx")
SENSITIVE_NAME_MARKERS = ("auth", "credential", "secret", "token")
SENSITIVE_METADATA_SUFFIXES = ("", ".json", ".jsonl", ".toml", ".yaml", ".yml", ".txt")
def __init__(self) -> None:
self.omitted_counts: dict[str, int] = {}
def _omit(self, category: str) -> bool:
self.omitted_counts[category] = self.omitted_counts.get(category, 0) + 1
return True
def category(self, path: Path) -> str | None:
name = path.name.lower()
if path.is_symlink():
return "symlink"
if path.is_dir() and name in self.OMITTED_DIRECTORIES:
return "runtime_or_sensitive_directory"
if name in self.OMITTED_EXACT_FILES or name.startswith(".env.") or name.endswith(".env"):
return "environment_or_auth_file"
if any(marker in name for marker in self.SENSITIVE_NAME_MARKERS) and path.suffix.lower() in (
self.SENSITIVE_METADATA_SUFFIXES
):
return "credential_file"
if name.startswith("state."):
return "state_database"
if "backup" in name or ".bak-" in name or name.endswith((".bak", ".old")):
return "backup"
if name.endswith(self.STATE_SUFFIXES):
return "state_database"
if name.endswith(self.SECRET_SUFFIXES):
return "credential_file"
if name.endswith(".lock"):
return "lock"
return None
def __call__(self, directory: str, names: list[str]) -> set[str]:
ignored: set[str] = set()
parent = Path(directory)
for name in names:
category = self.category(parent / name)
if category and self._omit(category):
ignored.add(name)
return ignored
def _safe_name(value: str, fallback: str) -> str:
normalized = re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-.")
return (normalized or fallback)[:48]
def register_temp_root(path: Path) -> None:
_ACTIVE_TEMP_ROOTS.add(path)
def unregister_temp_root(path: Path | None) -> None:
if path is not None:
_ACTIVE_TEMP_ROOTS.discard(path)
def temp_path_absent(path: Path | None) -> bool:
return path is None or not os.path.lexists(path)
def copy_profile(
*,
run_id: str,
prompt_id: str,
live_profile: Path | None = None,
) -> tuple[Path, dict[str, Any]]:
live_profile = live_profile or LIVE_PROFILE
if not live_profile.is_dir():
raise CheckpointError(f"live leoclean profile does not exist: {live_profile}")
unique = uuid.uuid4().hex[:12]
prefix = f"leo-clone-checkpoint-{_safe_name(run_id, 'run')}-{_safe_name(prompt_id, 'prompt')}-{unique}-"
temp_root = Path(tempfile.mkdtemp(prefix=prefix))
register_temp_root(temp_root)
temp_root.chmod(0o700)
target = temp_root / "profile"
policy = ProfileCopyPolicy()
try:
shutil.copytree(live_profile, target, symlinks=False, ignore=policy)
audit = audit_temp_profile(target, policy)
if not audit["passes"]:
raise CheckpointError("temporary profile retained a forbidden state or credential artifact")
except BaseException:
remove_tree(temp_root)
raise
return target, audit
def copy_ephemeral_model_auth(
temp_profile: Path,
*,
live_profile: Path | None = None,
) -> dict[str, Any]:
live_profile = live_profile or LIVE_PROFILE
source = live_profile / "auth.json"
target = temp_profile / "auth.json"
if not source.is_file() or source.is_symlink():
raise CheckpointError(f"live model auth file is unavailable or unsafe: {source}")
if target.exists() or target.is_symlink():
raise CheckpointError(f"temporary model auth target already exists: {target}")
shutil.copyfile(source, target)
target.chmod(0o600)
return {
"mode": "ephemeral_file_copy",
"source_path": str(source),
"temporary_path": str(target),
"temporary_mode": oct(target.stat().st_mode & 0o777),
"source_contents_recorded": False,
"source_fingerprint_recorded": False,
}
def load_ephemeral_provider_environment(temp_profile: Path) -> tuple[dict[str, str], dict[str, Any]]:
"""Resolve only env-backed credentials for the profile's configured provider."""
try:
import yaml
config = yaml.safe_load((temp_profile / "config.yaml").read_text(encoding="utf-8")) or {}
auth = json.loads((temp_profile / "auth.json").read_text(encoding="utf-8"))
except (OSError, ValueError, TypeError) as exc:
raise CheckpointError(f"cannot resolve temporary provider credential binding: {exc}") from exc
model_config = config.get("model") if isinstance(config, dict) else None
configured_provider = model_config.get("provider") if isinstance(model_config, dict) else None
provider = str(configured_provider or auth.get("active_provider") or "").strip().lower()
pool = auth.get("credential_pool") if isinstance(auth, dict) else None
entries = pool.get(provider, []) if isinstance(pool, dict) else []
if not isinstance(entries, list):
raise CheckpointError(f"temporary credential pool for provider {provider!r} is not a list")
environment: dict[str, str] = {}
sources: list[str] = []
ordered_entries = sorted(
(entry for entry in entries if isinstance(entry, dict)),
key=lambda entry: int(entry.get("priority", 0) or 0),
)
for entry in ordered_entries:
source = str(entry.get("source") or "")
if not source.startswith("env:"):
continue
name = source.removeprefix("env:").strip()
if not SAFE_ENV_NAME_RE.fullmatch(name):
raise CheckpointError(f"unsafe provider credential environment name: {name!r}")
if not (name.endswith("_API_KEY") or name.endswith("_TOKEN")):
raise CheckpointError(f"provider credential environment name is outside the allowlist: {name!r}")
value = str(entry.get("access_token") or entry.get("api_key") or "")
if value and name not in environment:
environment[name] = value
sources.append(source)
return environment, {
"configured_provider": provider or None,
"environment_names": sorted(environment),
"bound_credential_count": len(environment),
"sources": sorted(sources),
"credential_contents_recorded": False,
"credential_fingerprints_recorded": False,
}
def audit_temp_profile(profile: Path, policy: ProfileCopyPolicy | None = None) -> dict[str, Any]:
policy = policy or ProfileCopyPolicy()
forbidden_categories: dict[str, int] = {}
file_count = 0
byte_count = 0
for path in profile.rglob("*"):
category = policy.category(path)
if category:
forbidden_categories[category] = forbidden_categories.get(category, 0) + 1
if path.is_file() and not path.is_symlink():
file_count += 1
byte_count += path.stat().st_size
return {
"passes": not forbidden_categories,
"copied_file_count": file_count,
"copied_bytes": byte_count,
"omitted_counts_by_category": dict(sorted(policy.omitted_counts.items())),
"forbidden_artifact_counts_by_category": dict(sorted(forbidden_categories.items())),
"secret_contents_read_or_recorded": False,
}
def _detach_and_write(path: Path, content: str, *, mode: int) -> None:
if path.is_symlink():
path.unlink()
elif path.exists() and not path.is_file():
raise CheckpointError(f"temporary bridge path is not a regular file: {path}")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
path.chmod(mode)
def build_forced_bridge_wrapper(
*,
kb_tool: Path,
tool_log: Path,
container: str,
database: str,
run_nonce: str,
) -> str:
python = str(Path(sys.executable).resolve())
return f"""#!/bin/bash
set -uo pipefail
KB_TOOL={shlex.quote(str(kb_tool))}
TOOL_LOG={shlex.quote(str(tool_log))}
TARGET_CONTAINER={shlex.quote(container)}
TARGET_DATABASE={shlex.quote(database)}
RUN_NONCE={shlex.quote(run_nonce)}
PYTHON={shlex.quote(python)}
DOCKER={shlex.quote(SYSTEM_DOCKER)}
export PATH={shlex.quote(SYSTEM_EXEC_PATH)}
INVOCATION_ID="$($PYTHON -c 'import uuid; print(uuid.uuid4())')"
DB_RECEIPT="$($DOCKER exec "$TARGET_CONTAINER" psql -U postgres -d "$TARGET_DATABASE" -At -q -v ON_ERROR_STOP=1 -c "select jsonb_build_object('current_database', current_database(), 'system_identifier', (select system_identifier::text from pg_control_system()), 'transaction_read_only', current_setting('transaction_read_only'))::text")"
if [[ -z "$DB_RECEIPT" ]]; then
exit 125
fi
$PYTHON - "$TOOL_LOG" "$RUN_NONCE" "$INVOCATION_ID" "$TARGET_CONTAINER" "$TARGET_DATABASE" "$DB_RECEIPT" "$@" <<'PY'
import json
import sys
from datetime import datetime, timezone
path, run_nonce, invocation_id, container, database, db_receipt, *argv = sys.argv[1:]
event = {{
"phase": "start",
"at_utc": datetime.now(timezone.utc).isoformat(),
"invocation_id": invocation_id,
"run_nonce": run_nonce,
"container": container,
"database": database,
"database_identity": json.loads(db_receipt),
"argv": argv,
}}
with open(path, "a", encoding="utf-8") as handle:
handle.write(json.dumps(event, sort_keys=True) + "\\n")
PY
$PYTHON "$KB_TOOL" --local --container "$TARGET_CONTAINER" --db "$TARGET_DATABASE" "$@"
STATUS=$?
$PYTHON - "$TOOL_LOG" "$RUN_NONCE" "$INVOCATION_ID" "$TARGET_CONTAINER" "$TARGET_DATABASE" "$STATUS" <<'PY'
import json
import sys
from datetime import datetime, timezone
path, run_nonce, invocation_id, container, database, status = sys.argv[1:]
event = {{
"phase": "end",
"at_utc": datetime.now(timezone.utc).isoformat(),
"invocation_id": invocation_id,
"run_nonce": run_nonce,
"container": container,
"database": database,
"returncode": int(status),
}}
with open(path, "a", encoding="utf-8") as handle:
handle.write(json.dumps(event, sort_keys=True) + "\\n")
PY
exit "$STATUS"
"""
def inject_skill_binding(skill_text: str, binding: str) -> str:
if not skill_text.startswith("---\n"):
return binding.rstrip() + "\n\n" + skill_text
frontmatter_end = skill_text.find("\n---\n", 4)
if frontmatter_end < 0:
raise CheckpointError("temporary teleo-kb bridge skill has unterminated YAML frontmatter")
insertion = frontmatter_end + len("\n---\n")
return skill_text[:insertion] + "\n" + binding.rstrip() + "\n\n" + skill_text[insertion:]
def patch_temp_bridge(temp_profile: Path, container: str, database: str) -> dict[str, Any]:
wrapper = temp_profile / "bin" / "teleo-kb"
kb_tool = temp_profile / "bin" / "kb_tool.py"
bridge_skill = temp_profile / "skills" / "teleo-kb-bridge" / "SKILL.md"
tool_log = temp_profile / "checkpoint-tool-calls.jsonl"
run_nonce = uuid.uuid4().hex
if not wrapper.exists() and not wrapper.is_symlink():
raise CheckpointError(f"temporary teleo-kb wrapper is missing: {wrapper}")
if not kb_tool.exists():
raise CheckpointError(f"temporary kb_tool.py is missing: {kb_tool}")
if not bridge_skill.exists():
raise CheckpointError(f"temporary teleo-kb bridge skill is missing: {bridge_skill}")
skill_text = bridge_skill.read_text(encoding="utf-8")
temporary_path = str(wrapper)
for live_wrapper in {
"/home/teleo/.hermes/profiles/leoclean/bin/teleo-kb",
str(LIVE_PROFILE / "bin" / "teleo-kb"),
}:
skill_text = skill_text.replace(live_wrapper, "teleo-kb")
binding = f"""
## Disposable Checkpoint Binding
This temporary profile is bound only to `{container}` / `{database}`. For all
KB reads in this session, start a terminal call directly with `teleo-kb`; the
checkpoint PATH resolves that name only to `{temporary_path}`. Run exactly one
KB command per terminal call. Do not define shell variables or use `exec`,
pipes, redirects, shell operators, `docker`, `psql`, cloud database tools, or
wrappers under the live profile. This checkpoint exposes only read-only bridge
verbs and records every invocation in its receipt.
End the final answer with one machine-readable line using the full proposal
UUID and the row state you observed:
`KB_STATE: {{"proposal_id":"<uuid>","status":"pending_review|approved|applied","applied":false}}`
"""
skill_text = inject_skill_binding(skill_text, binding.lstrip())
wrapper_text = build_forced_bridge_wrapper(
kb_tool=kb_tool,
tool_log=tool_log,
container=container,
database=database,
run_nonce=run_nonce,
)
try:
descriptor = os.open(
tool_log,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
0o600,
)
except FileExistsError as exc:
raise CheckpointError("temporary tool log must not pre-exist") from exc
os.close(descriptor)
_detach_and_write(wrapper, wrapper_text, mode=0o700)
_detach_and_write(bridge_skill, skill_text, mode=0o600)
return {
"wrapper_path": str(wrapper),
"bridge_skill_path": str(bridge_skill),
"tool_log_path": str(tool_log),
"wrapper_sha256": sha256_bytes(wrapper_text.encode()),
"bridge_skill_sha256": sha256_bytes(skill_text.encode()),
"forced_container": container,
"forced_database": database,
"run_nonce": run_nonce,
"patched_files": [str(wrapper), str(bridge_skill)],
}
def bridge_hashes(profile: Path | None = None) -> dict[str, str | None]:
profile = profile or LIVE_PROFILE
paths = {
"wrapper": profile / "bin" / "teleo-kb",
"bridge_skill": profile / "skills" / "teleo-kb-bridge" / "SKILL.md",
}
result: dict[str, str | None] = {}
for name, path in paths.items():
try:
result[name] = sha256_bytes(path.read_bytes())
except OSError:
result[name] = None
return result
@contextmanager
def gateway_environment(
temp_profile: Path,
*,
chat_id: str,
user_id: str,
provider_environment: dict[str, str] | None = None,
) -> Iterator[None]:
updates = {
"HERMES_HOME": str(temp_profile),
"HOME": str(LIVE_PROFILE.parents[2]),
"PATH": f"{temp_profile / 'bin'}:{os.environ.get('PATH', '')}",
"TELEO_KB_MODE": "local",
"TELEGRAM_ALLOWED_CHATS": chat_id,
"TELEGRAM_ALLOWED_USERS": user_id,
"TELEGRAM_ALLOW_ALL_USERS": "false",
"GATEWAY_ALLOW_ALL_USERS": "false",
}
provider_environment = provider_environment or {}
collisions = sorted(set(updates) & set(provider_environment))
if collisions:
raise CheckpointError(f"provider credential environment collides with safety settings: {collisions}")
updates.update(provider_environment)
old = {key: os.environ.get(key) for key in updates}
original_sys_path = list(sys.path)
try:
os.environ.update(updates)
sys.path.insert(0, str(AGENT_ROOT))
yield
finally:
sys.path[:] = original_sys_path
for key, value in old.items():
if value is None:
os.environ.pop(key, None)
else:
os.environ[key] = value
async def _close_runner(runner: Any) -> dict[str, Any]:
for method_name in ("aclose", "close"):
method = getattr(runner, method_name, None)
if not callable(method):
continue
try:
result = method()
if inspect.isawaitable(result):
await asyncio.wait_for(result, timeout=15)
return {"attempted": True, "method": method_name, "ok": True}
except Exception as exc: # pragma: no cover - depends on live runner implementation
return {"attempted": True, "method": method_name, "ok": False, "error": str(exc)}
return {"attempted": False, "method": None, "ok": True}
def transcript_tool_trace(
messages: list[dict[str, Any]],
*,
content_limit: int = 4000,
argument_limit: int = 2048,
) -> list[dict[str, Any]]:
"""Retain bounded tool-call evidence without copying the full conversation."""
trace: list[dict[str, Any]] = []
for message in messages:
role = message.get("role")
tool_calls = message.get("tool_calls")
if role == "assistant" and isinstance(tool_calls, list):
for call in tool_calls:
if not isinstance(call, dict):
continue
function = call.get("function") if isinstance(call.get("function"), dict) else {}
raw_arguments = str(function.get("arguments") or call.get("arguments") or "")
trace.append(
{
"phase": "call",
"tool_call_id": call.get("id"),
"tool_name": function.get("name") or call.get("name"),
"arguments": redact_text(raw_arguments)[:argument_limit],
"arguments_truncated": len(raw_arguments) > argument_limit,
}
)
elif role == "tool":
raw_content = str(message.get("content") or "")
content = redact_text(raw_content)
trace.append(
{
"phase": "result",
"tool_call_id": message.get("tool_call_id"),
"tool_name": message.get("tool_name") or message.get("name"),
"content": content[:content_limit],
"content_truncated": len(raw_content) > content_limit or len(content) > content_limit,
}
)
return redact_value(trace)
def transcript_terminal_attempt_summary(events: list[dict[str, Any]]) -> dict[str, Any]:
terminal_calls = {
str(event.get("tool_call_id")): event
for event in events
if event.get("phase") == "call" and event.get("tool_name") == "terminal"
}
result_content = {
str(event.get("tool_call_id")): str(event.get("content") or "")
for event in events
if event.get("phase") == "result"
}
known_exit_codes: dict[str, int] = {}
for call_id in terminal_calls:
match = re.search(r'"exit_code"\s*:\s*(-?\d+)', result_content.get(call_id, ""))
if match:
known_exit_codes[call_id] = int(match.group(1))
rejected = sorted(call_id for call_id, exit_code in known_exit_codes.items() if exit_code == 126)
nonzero = sorted(call_id for call_id, exit_code in known_exit_codes.items() if exit_code != 0)
return {
"call_count": len(terminal_calls),
"known_exit_codes": known_exit_codes,
"rejected_call_ids": rejected,
"rejected_call_count": len(rejected),
"nonzero_call_ids": nonzero,
"nonzero_call_count": len(nonzero),
"result_code_unknown_call_count": len(terminal_calls) - len(known_exit_codes),
}
class BridgeArgumentParser(argparse.ArgumentParser):
def error(self, message: str) -> None:
raise CheckpointError(message)
def exit(self, status: int = 0, message: str | None = None) -> None:
raise CheckpointError(message or f"argument parser exited with status {status}")
def _uuid_argument(value: str) -> str:
try:
return str(uuid.UUID(value))
except ValueError as exc:
raise argparse.ArgumentTypeError("expected a UUID") from exc
def validate_read_only_bridge_argv(argv: list[str]) -> None:
parser = BridgeArgumentParser(add_help=False)
subparsers = parser.add_subparsers(dest="command", required=True)
def command(name: str) -> argparse.ArgumentParser:
item = subparsers.add_parser(name, add_help=False)
item.add_argument("--format", choices=("markdown", "json"), default="markdown")
return item
for name in ("search", "context"):
item = command(name)
item.add_argument("query")
item.add_argument("--limit", type=int, default=5)
if name == "search":
item.add_argument("--context-limit", type=int, default=5)
else:
item.add_argument("--context-limit", type=int, default=6)
show = command("show")
show.add_argument("claim_id", type=_uuid_argument)
for name in ("evidence", "edges"):
item = command(name)
item.add_argument("claim_id", type=_uuid_argument)
item.add_argument("--limit", type=int, default=12)
list_proposals = command("list-proposals")
list_proposals.add_argument(
"--status",
choices=("pending_review", "approved", "applied", "canceled", "all"),
default="pending_review",
)
list_proposals.add_argument("--limit", type=int, default=10)
search_proposals = command("search-proposals")
search_proposals.add_argument("query")
search_proposals.add_argument(
"--status",
choices=("pending_review", "approved", "applied", "canceled", "all"),
default="all",
)
search_proposals.add_argument("--limit", type=int, default=10)
show_proposal = command("show-proposal")
show_proposal.add_argument("proposal_id", type=_uuid_argument)
command("decision-matrix-status")
parsed = parser.parse_args(argv)
for name, value in vars(parsed).items():
if name.endswith("limit") and not 1 <= value <= 100:
raise CheckpointError(f"{name.replace('_', '-')} must be between 1 and 100")
def runner_tool_trace(runner: Any, session_key: str) -> dict[str, Any]:
entry = getattr(getattr(runner, "session_store", None), "_entries", {}).get(session_key)
session_id = getattr(entry, "session_id", None)
if not session_id:
return {"session_id": None, "events": [], "event_count": 0}
messages = runner.session_store.load_transcript(session_id)
events = transcript_tool_trace(messages)
return {"session_id": session_id, "events": events, "event_count": len(events)}
def restricted_bridge_terminal_handler(
wrapper: Path,
temp_profile: Path,
tool_args: dict[str, Any],
**_kwargs: Any,
) -> str:
command = str(tool_args.get("command") or "")
if len(command) > 8192:
return json.dumps({"output": "", "exit_code": 126, "error": "checkpoint command is too long"})
if tool_args.get("background") or tool_args.get("pty") or tool_args.get("workdir"):
return json.dumps(
{"output": "", "exit_code": 126, "error": "background, PTY, and workdir overrides are disabled"}
)
try:
argv = shlex.split(command)
except ValueError as exc:
return json.dumps({"output": "", "exit_code": 126, "error": f"invalid command quoting: {exc}"})
while argv and argv[-1] in HARMLESS_TRAILING_REDIRECT_TOKENS:
argv.pop()
command_path = f"{temp_profile / 'bin'}:{os.environ.get('PATH', '')}"
try:
exact_wrapper = wrapper.resolve(strict=True)
executable = shutil.which(argv[0], path=command_path) if argv else None
invoked = Path(executable).resolve(strict=True) if executable else None
except OSError:
invoked = None
exact_wrapper = wrapper.resolve(strict=False)
if invoked != exact_wrapper:
return json.dumps(
{"output": "", "exit_code": 126, "error": "only the clone-bound teleo-kb wrapper is available"}
)
if len(argv) < 2 or argv[1] not in READ_ONLY_BRIDGE_COMMANDS:
return json.dumps(
{"output": "", "exit_code": 126, "error": "only read-only Teleo KB bridge commands are available"}
)
try:
validate_read_only_bridge_argv(argv[1:])
except CheckpointError as exc:
return json.dumps({"output": "", "exit_code": 126, "error": f"invalid read-only KB arguments: {exc}"})
timeout = min(max(int(tool_args.get("timeout") or 60), 1), 120)
child_environment = {
"HOME": str(temp_profile),
"PATH": command_path,
"TELEO_KB_MODE": "local",
}
try:
proc = subprocess.run(
argv,
cwd=temp_profile,
env=child_environment,
text=True,
capture_output=True,
check=False,
timeout=timeout,
)
except subprocess.TimeoutExpired:
return json.dumps({"output": "", "exit_code": 124, "error": f"bridge command exceeded {timeout}s"})
return json.dumps(
{
"output": redact_text(proc.stdout),
"exit_code": proc.returncode,
"error": redact_text(proc.stderr) or None,
}
)
def install_checkpoint_tool_surface(temp_profile: Path) -> dict[str, Any]:
import tools.skills_tool
import tools.terminal_tool # noqa: F401
import toolsets
from hermes_cli import tools_config
from tools.registry import registry
toolset_name = "checkpoint-kb-readonly"
allowed_tools = ["skills_list", "skill_view", "terminal"]
toolsets.TOOLSETS[toolset_name] = {
"description": "Read-only clone-bound KB checkpoint tools",
"tools": allowed_tools,
"includes": [],
}
tools_config._get_platform_tools = lambda *_args, **_kwargs: {toolset_name}
missing_tools = sorted(name for name in allowed_tools if name not in registry._tools)
if missing_tools:
raise CheckpointError(f"required checkpoint tools are not registered: {missing_tools}")
allowed_entries = {name: registry._tools[name] for name in allowed_tools}
registry._tools.clear()
registry._tools.update(allowed_entries)
terminal_entry = registry._tools.get("terminal")
if terminal_entry is None:
raise CheckpointError("Hermes terminal tool is not registered")
wrapper = temp_profile / "bin" / "teleo-kb"
terminal_entry.handler = lambda tool_args, **kwargs: restricted_bridge_terminal_handler(
wrapper,
temp_profile,
tool_args,
**kwargs,
)
return {
"toolsets": [toolset_name],
"allowed_tools": allowed_tools,
"actual_registry_tools": sorted(registry._tools),
"actual_registry_tool_types": {
name: f"{entry.__class__.__module__}.{entry.__class__.__name__}" for name, entry in registry._tools.items()
},
"actual_registry_tool_fields": {
name: sorted(vars(entry)) if hasattr(entry, "__dict__") else [] for name, entry in registry._tools.items()
},
"actual_registry_tool_schema_sha256": {
name: json_sha256(
{
field: redact_text(repr(getattr(entry, field)))[:4096]
for field in ("name", "description", "parameters", "schema", "input_schema")
if hasattr(entry, field)
}
)
for name, entry in registry._tools.items()
},
"read_only_bridge_commands": sorted(READ_ONLY_BRIDGE_COMMANDS),
"send_message_tool_enabled": "send_message" in registry._tools,
"terminal_restricted_to_clone_wrapper": True,
"terminal_subprocess_inherits_provider_credentials": False,
}
async def invoke_gateway(
args: argparse.Namespace,
temp_profile: Path,
*,
provider_environment: dict[str, str] | None = None,
) -> dict[str, Any]:
with gateway_environment(
temp_profile,
chat_id=args.chat_id,
user_id=args.user_id,
provider_environment=provider_environment,
):
from gateway.config import Platform
from gateway.platforms.base import MessageEvent, MessageType
from gateway.run import GatewayRunner, _resolve_gateway_model, _resolve_runtime_agent_kwargs
from gateway.session import SessionSource
tool_surface = install_checkpoint_tool_surface(temp_profile)
source = SessionSource(
platform=Platform.TELEGRAM,
chat_id=args.chat_id,
chat_name="Leo",
chat_type="group",
user_id=args.user_id,
user_name=args.user_name,
)
runner = GatewayRunner()
adapters = getattr(runner, "adapters", None)
if not isinstance(adapters, dict):
raise CheckpointError("GatewayRunner adapters could not be verified as a mapping")
tool_surface["gateway_adapters_verified_mapping"] = True
tool_surface["gateway_adapter_names"] = sorted(str(name) for name in adapters)
tool_surface["gateway_adapter_count"] = len(adapters)
gateway_model = _resolve_gateway_model()
runtime = _resolve_runtime_agent_kwargs()
turn_route = runner._resolve_turn_agent_config(args.prompt, gateway_model, runtime)
turn_runtime = turn_route.get("runtime") or {}
session_key = runner._session_key_for_source(source)
authorized = bool(runner._is_user_authorized(source))
result: dict[str, Any] = {
"runner_class": f"{runner.__class__.__module__}.{runner.__class__.__name__}",
"session_key": session_key,
"authorized": authorized,
"authorization_mode": "harness_exact_chat_and_user",
"handler_invoked": False,
"posted_to_telegram": False,
"model_free_fallback_used": False,
"tool_surface": tool_surface,
"routing": {
"gateway_model": gateway_model,
"turn_model": turn_route.get("model"),
"provider": turn_runtime.get("provider"),
"api_mode": turn_runtime.get("api_mode"),
"base_url_host": urlsplit(str(turn_runtime.get("base_url") or "")).hostname,
"api_key_present": bool(turn_runtime.get("api_key")),
"credential_pool_present": turn_runtime.get("credential_pool") is not None,
},
}
try:
if not authorized:
raise CheckpointError("Telegram-shaped source is not authorized by the copied live profile")
event = MessageEvent(
text=args.prompt,
message_type=MessageType.TEXT,
source=source,
message_id=f"clone-checkpoint-{args.run_id}-{args.prompt_id}",
)
result["started_at_utc"] = utc_now()
result["handler_invoked"] = True
reply = await runner._handle_message(event)
result["ended_at_utc"] = utc_now()
result["reply"] = reply if isinstance(reply, str) else str(reply or "")
result["transcript_tool_trace"] = runner_tool_trace(runner, session_key)
result["transcript_terminal_attempts"] = transcript_terminal_attempt_summary(
result["transcript_tool_trace"]["events"]
)
return result
finally:
result["runner_cleanup"] = await _close_runner(runner)
def process_group_alive(process: Any) -> bool:
pid = getattr(process, "pid", None)
if not pid:
return False
try:
os.killpg(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
def _signal_gateway_process_group(process: Any, signum: signal.Signals) -> None:
try:
os.killpg(process.pid, signum)
except ProcessLookupError:
if process.is_alive():
if signum == signal.SIGKILL:
process.kill()
else:
process.terminate()
def _wait_for_process_group_exit(process: Any, timeout: float) -> bool:
deadline = time.monotonic() + timeout
while process_group_alive(process) and time.monotonic() < deadline:
time.sleep(0.02)
return not process_group_alive(process)
def terminate_gateway_child(process: Any, *, grace_seconds: float = 2.0) -> dict[str, Any]:
attempted = bool(process and (process.is_alive() or process_group_alive(process)))
terminated = False
killed = False
if attempted:
_signal_gateway_process_group(process, signal.SIGTERM)
process.join(timeout=grace_seconds)
group_exited = _wait_for_process_group_exit(process, grace_seconds)
terminated = not process.is_alive() and group_exited
if process and (process.is_alive() or process_group_alive(process)):
_signal_gateway_process_group(process, signal.SIGKILL)
process.join(timeout=grace_seconds)
_wait_for_process_group_exit(process, grace_seconds)
killed = True
return {
"attempted": attempted,
"terminated_after_sigterm": terminated,
"sigkill_used": killed,
"alive_after_cleanup": bool(process and process.is_alive()),
"process_group_alive_after_cleanup": bool(process and process_group_alive(process)),
"exitcode": getattr(process, "exitcode", None),
}
def cleanup_active_gateway_children() -> dict[int | None, dict[str, Any]]:
results: dict[int | None, dict[str, Any]] = {}
for process in tuple(_ACTIVE_GATEWAY_CHILDREN):
result = terminate_gateway_child(process)
results[getattr(process, "pid", None)] = result
if not result["alive_after_cleanup"] and not result["process_group_alive_after_cleanup"]:
_ACTIVE_GATEWAY_CHILDREN.discard(process)
return results
def _gateway_child_entry(
args: argparse.Namespace,
temp_profile: Path,
provider_environment: dict[str, str],
result_path: Path,
readiness_path: Path,
) -> None:
os.setsid()
for signum in (signal.SIGINT, signal.SIGTERM, getattr(signal, "SIGHUP", None)):
if signum is not None:
signal.signal(signum, signal.SIG_DFL)
write_report(
readiness_path,
{"pid": os.getpid(), "process_group_id": os.getpgrp(), "ready_at_utc": utc_now()},
)
if hasattr(signal, "pthread_sigmask"):
signal.pthread_sigmask(
signal.SIG_UNBLOCK,
{item for item in (signal.SIGINT, signal.SIGTERM, getattr(signal, "SIGHUP", None)) if item is not None},
)
retained_environment = {
key: value
for key, value in os.environ.items()
if key in {"PATH", "LANG", "LC_ALL", "LC_CTYPE", "TZ", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "TMPDIR"}
}
os.environ.clear()
os.environ.update(retained_environment)
os.environ["HOME"] = "/home/teleo"
try:
result = asyncio.run(
invoke_gateway(
args,
temp_profile,
provider_environment=provider_environment,
)
)
except BaseException as exc:
result = {
"authorized": False,
"handler_invoked": False,
"posted_to_telegram": False,
"model_free_fallback_used": False,
"handler_error": {"type": type(exc).__name__, "message": str(exc)},
}
write_report(result_path, result, secret_values=tuple(provider_environment.values()))
async def invoke_gateway_subprocess(
args: argparse.Namespace,
temp_profile: Path,
*,
provider_environment: dict[str, str] | None = None,
) -> dict[str, Any]:
if "fork" not in multiprocessing.get_all_start_methods():
raise CheckpointError("the live VPS gateway checkpoint requires multiprocessing fork support")
context = multiprocessing.get_context("fork")
result_path = temp_profile / ".checkpoint-gateway-result.json"
readiness_path = temp_profile / ".checkpoint-gateway-ready.json"
result_path.unlink(missing_ok=True)
readiness_path.unlink(missing_ok=True)
process = context.Process(
target=_gateway_child_entry,
args=(args, temp_profile, dict(provider_environment or {}), result_path, readiness_path),
name=f"leo-clone-checkpoint-{_safe_name(args.prompt_id, 'prompt')}",
)
watched_signals = {
item for item in (signal.SIGINT, signal.SIGTERM, getattr(signal, "SIGHUP", None)) if item is not None
}
previous_mask = None
if hasattr(signal, "pthread_sigmask"):
previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, watched_signals)
try:
process.start()
_ACTIVE_GATEWAY_CHILDREN.add(process)
finally:
if previous_mask is not None:
signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask)
readiness: dict[str, Any] | None = None
readiness_deadline = asyncio.get_running_loop().time() + 30
while process.is_alive() and asyncio.get_running_loop().time() < readiness_deadline:
if readiness_path.is_file():
try:
readiness = json.loads(readiness_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
readiness = None
if readiness:
break
await asyncio.sleep(0.05)
if readiness is None and readiness_path.is_file():
try:
readiness = json.loads(readiness_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
readiness = None
deadline = asyncio.get_running_loop().time() + args.turn_timeout
timed_out = False
termination = {
"attempted": False,
"terminated_after_sigterm": False,
"sigkill_used": False,
"alive_after_cleanup": False,
"process_group_alive_after_cleanup": False,
"exitcode": None,
}
try:
while readiness is not None and process.is_alive() and asyncio.get_running_loop().time() < deadline:
await asyncio.sleep(0.1)
if process.is_alive() and (readiness is None or asyncio.get_running_loop().time() >= deadline):
timed_out = True
termination = terminate_gateway_child(process)
else:
process.join(timeout=1)
result = None
if result_path.is_file():
try:
result = json.loads(result_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
result = None
if result is None:
result = {
"authorized": False,
"handler_invoked": False,
"posted_to_telegram": False,
"model_free_fallback_used": False,
"handler_error": {
"type": "TimeoutError" if timed_out else "GatewayChildError",
"message": (
f"gateway child exceeded {args.turn_timeout}s and was terminated"
if timed_out
else f"gateway child exited without a result (exitcode={process.exitcode})"
),
},
}
result["child_process"] = {
"pid": process.pid,
"timed_out": timed_out,
"exitcode": process.exitcode,
"termination": termination,
"alive_after_readback": process.is_alive(),
"process_group_alive_after_readback": process_group_alive(process),
"result_transport": "private_temp_file",
"readiness": readiness,
"readiness_verified": bool(
readiness and readiness.get("pid") == process.pid and readiness.get("process_group_id") == process.pid
),
}
return result
finally:
if process.is_alive() or process_group_alive(process):
terminate_gateway_child(process)
if not process.is_alive() and not process_group_alive(process):
_ACTIVE_GATEWAY_CHILDREN.discard(process)
def read_tool_proof(
path: Path | None,
container: str,
database: str,
*,
run_nonce: str,
database_identity: dict[str, Any],
require_database_read_only: bool = True,
) -> dict[str, Any]:
events: list[dict[str, Any]] = []
parse_errors: list[str] = []
if path and path.exists():
for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
if not line.strip():
continue
try:
event = json.loads(line)
except json.JSONDecodeError as exc:
parse_errors.append(f"line {number}: {exc}")
continue
events.append(event)
starts = [event for event in events if event.get("phase") == "start"]
end_events = [event for event in events if event.get("phase") == "end"]
start_ids = [event.get("invocation_id") for event in starts]
end_ids = [event.get("invocation_id") for event in end_events]
duplicate_ids = sorted(
str(item) for item in set(start_ids + end_ids) if start_ids.count(item) > 1 or end_ids.count(item) > 1
)
ends = {event.get("invocation_id"): event for event in end_events}
invocations = []
for event in events:
if event.get("phase") != "start":
continue
end = ends.get(event.get("invocation_id"), {})
invocations.append(
{
"invocation_id": event.get("invocation_id"),
"started_at_utc": event.get("at_utc"),
"ended_at_utc": end.get("at_utc"),
"container": event.get("container"),
"database": event.get("database"),
"run_nonce": event.get("run_nonce"),
"database_identity": event.get("database_identity"),
"argv": [redact_text(str(item))[:512] for item in event.get("argv", [])[:32]],
"returncode": end.get("returncode"),
"end_run_nonce": end.get("run_nonce"),
}
)
expected_system_identifier = str(database_identity.get("system_identifier") or "")
complete_pairing = (
not duplicate_ids and len(starts) == len(end_events) and set(start_ids) == set(end_ids) and not parse_errors
)
all_bound = bool(invocations) and all(
item["container"] == container
and item["database"] == database
and item["run_nonce"] == run_nonce
and item["end_run_nonce"] == run_nonce
and str((item["database_identity"] or {}).get("current_database")) == database
and str((item["database_identity"] or {}).get("system_identifier")) == expected_system_identifier
and (
not require_database_read_only
or str((item["database_identity"] or {}).get("transaction_read_only")) == "on"
)
for item in invocations
)
return {
"parse_errors": parse_errors,
"event_count": len(events),
"duplicate_invocation_ids": duplicate_ids,
"complete_start_end_pairing": complete_pairing,
"invocations": invocations,
"invocation_count": len(invocations),
"all_bound_to_supplied_target": complete_pairing and all_bound,
"database_read_only_required": require_database_read_only,
"all_completed_successfully": complete_pairing
and bool(invocations)
and all(item["returncode"] == 0 for item in invocations),
}
def remove_tree(path: Path | None) -> bool:
if path is None or temp_path_absent(path):
unregister_temp_root(path)
return True
try:
shutil.rmtree(path)
except Exception:
for root, directories, files in os.walk(path, topdown=False):
for name in files:
item = Path(root) / name
try:
item.chmod(0o600)
item.unlink()
except OSError:
pass
for name in directories:
item = Path(root) / name
try:
item.chmod(0o700)
item.rmdir()
except OSError:
pass
try:
path.chmod(0o700)
path.rmdir()
except OSError:
pass
absent = temp_path_absent(path)
if absent:
unregister_temp_root(path)
return absent
def cleanup_active_temp_roots() -> dict[str, bool]:
return {str(path): remove_tree(path) and temp_path_absent(path) for path in tuple(_ACTIVE_TEMP_ROOTS)}
def _termination_handler(signum: int, _frame: Any) -> None:
global _LAST_TERMINATION_SIGNAL
_LAST_TERMINATION_SIGNAL = signum
child_results = cleanup_active_gateway_children()
if all(
not result["alive_after_cleanup"] and not result["process_group_alive_after_cleanup"]
for result in child_results.values()
):
cleanup_active_temp_roots()
raise TerminationRequested(signum)
@contextmanager
def termination_cleanup_handlers() -> Iterator[None]:
watched = tuple(
item for item in (signal.SIGINT, signal.SIGTERM, getattr(signal, "SIGHUP", None)) if item is not None
)
previous: dict[signal.Signals, Any] = {}
try:
for signum in watched:
previous[signum] = signal.getsignal(signum)
signal.signal(signum, _termination_handler)
yield
finally:
child_results = cleanup_active_gateway_children()
if all(
not result["alive_after_cleanup"] and not result["process_group_alive_after_cleanup"]
for result in child_results.values()
):
cleanup_active_temp_roots()
for signum, handler in previous.items():
signal.signal(signum, handler)
def _uuid_values(value: Any) -> set[str]:
found: set[str] = set()
if isinstance(value, dict):
for key, item in value.items():
found.update(_uuid_values(key))
found.update(_uuid_values(item))
elif isinstance(value, list):
for item in value:
found.update(_uuid_values(item))
elif isinstance(value, str):
found.update(match.group(0).lower() for match in UUID_RE.finditer(value))
return found
def _state_semantics(proposal: dict[str, Any], expected_state: str) -> dict[str, bool]:
status_matches = proposal.get("status") == expected_state
has_reviewer = bool(proposal.get("reviewed_by_handle") or proposal.get("reviewed_by_agent_id"))
has_applier = bool(proposal.get("applied_by_handle") or proposal.get("applied_by_agent_id"))
if expected_state == "pending_review":
review_matches = proposal.get("reviewed_at") is None and not has_reviewer
apply_matches = proposal.get("applied_at") is None and not has_applier
elif expected_state == "approved":
review_matches = bool(proposal.get("reviewed_at")) and has_reviewer
apply_matches = proposal.get("applied_at") is None and not has_applier
else:
review_matches = bool(proposal.get("reviewed_at")) and has_reviewer
apply_matches = bool(proposal.get("applied_at")) and has_applier
return {
"status_matches": status_matches,
"review_fields_match": review_matches,
"apply_fields_match": apply_matches,
"all": status_matches and review_matches and apply_matches,
}
def select_expected_proposal(
*,
before: dict[str, Any],
after: dict[str, Any],
expected_state: str,
prompt: str,
reply: str,
explicit_proposal_id: str | None,
) -> dict[str, Any]:
before_by_id = {str(row.get("id")): row for row in before.get("proposals", [])}
candidates = [row for row in after.get("proposals", []) if row.get("status") == expected_state]
selected: list[dict[str, Any]] = []
method = "none"
if explicit_proposal_id:
selected = [row for row in candidates if str(row.get("id")) == explicit_proposal_id]
method = "explicit_proposal_id"
else:
combined_text = f"{prompt}\n{reply}".lower()
mentioned = {match.group(0).lower() for match in UUID_RE.finditer(combined_text)}
selected = [row for row in candidates if str(row.get("id", "")).lower() in mentioned]
if len(selected) == 1:
method = "prompt_or_reply_uuid"
else:
prefix_matches = [
row
for row in candidates
if re.search(rf"\b{re.escape(str(row.get('id', '')).lower()[:8])}\b", combined_text)
]
if len(prefix_matches) == 1:
selected = prefix_matches
method = "unique_prompt_or_reply_uuid_prefix"
else:
changed = [row for row in candidates if before_by_id.get(str(row.get("id"))) != row]
if len(changed) == 1:
selected = changed
method = "unique_new_or_changed_expected_state"
elif len(candidates) == 1:
selected = candidates
method = "unique_expected_state"
else:
selected = []
method = "ambiguous"
proposal = selected[0] if len(selected) == 1 else None
semantics = (
_state_semantics(proposal, expected_state)
if proposal
else {
"status_matches": False,
"review_fields_match": False,
"apply_fields_match": False,
"all": False,
}
)
return {
"selection_method": method,
"candidate_ids": [str(row.get("id")) for row in candidates],
"selected_proposal": proposal,
"selected_exactly_one": proposal is not None,
"state_semantics": semantics,
}
def validate_private_output_path(requested_output: Path) -> Path:
requested_output = requested_output.expanduser()
if requested_output.is_symlink():
raise CheckpointError("--output must not be a symbolic link")
try:
output_parent = requested_output.parent.resolve(strict=True)
except OSError as exc:
raise CheckpointError("--output parent must already exist") from exc
output = output_parent / requested_output.name
live_profile = LIVE_PROFILE.expanduser().resolve(strict=False)
if output == live_profile or live_profile in output.parents:
raise CheckpointError("--output must be outside the live leoclean profile")
parent_stat = output_parent.stat()
if not output_parent.is_dir() or parent_stat.st_uid != os.geteuid() or parent_stat.st_mode & 0o077:
raise CheckpointError("--output parent must be an existing private directory owned by the current user")
return output
def validate_args(args: argparse.Namespace) -> None:
if not SAFE_DOCKER_NAME_RE.fullmatch(args.container):
raise CheckpointError("--container must be an explicit Docker name, not a shell expression")
if args.container == PRODUCTION_CONTAINER:
raise CheckpointError("--container must not target the production teleo-pg container")
if not SAFE_DB_NAME_RE.fullmatch(args.db):
raise CheckpointError("--db must be an explicit Postgres database name")
if not args.prompt_id.strip() or not args.prompt.strip():
raise CheckpointError("--prompt-id and --prompt must be non-empty")
if args.expected_state not in EXPECTED_STATES:
raise CheckpointError(f"--expected-state must be one of {', '.join(EXPECTED_STATES)}")
if args.turn_timeout <= 0:
raise CheckpointError("--turn-timeout must be positive")
args.output = validate_private_output_path(args.output)
if args.proposal_id:
try:
args.proposal_id = str(uuid.UUID(args.proposal_id))
except ValueError as exc:
raise CheckpointError("--proposal-id must be a UUID") from exc
def _same_service(before: dict[str, Any] | None, after: dict[str, Any] | None) -> bool:
return bool(before and after and before.get("returncode") == 0 and before == after)
def _reply_mentions_proposal(reply: str, proposal: dict[str, Any] | None) -> bool:
if not proposal:
return False
proposal_id = str(proposal.get("id", "")).lower()
lowered = reply.lower()
return bool(proposal_id) and (proposal_id in lowered or proposal_id[:8] in lowered)
def tool_proof_read_exact_proposal(tool_proof: dict[str, Any], proposal: dict[str, Any] | None) -> bool:
if not proposal:
return False
expected = ["show-proposal", str(proposal.get("id") or "")]
return any(
invocation.get("argv") == expected
or (
invocation.get("argv", [])[:2] == expected
and invocation.get("argv", [])[2:] in (["--format", "json"], ["--format", "markdown"])
)
for invocation in tool_proof.get("invocations", [])
)
def parse_reply_state_receipt(reply: str) -> dict[str, Any]:
for line in reversed(reply.splitlines()):
prefix, separator, payload = line.strip().partition("KB_STATE:")
if prefix or not separator:
continue
try:
value = json.loads(payload.strip())
except json.JSONDecodeError as exc:
return {"valid": False, "error": f"invalid KB_STATE JSON: {exc}"}
if not isinstance(value, dict):
return {"valid": False, "error": "KB_STATE must be a JSON object"}
proposal_id = value.get("proposal_id")
status = value.get("status")
applied = value.get("applied")
try:
proposal_id = str(uuid.UUID(str(proposal_id)))
except ValueError:
return {"valid": False, "error": "KB_STATE proposal_id must be a full UUID"}
if status not in EXPECTED_STATES or not isinstance(applied, bool):
return {"valid": False, "error": "KB_STATE status/applied fields are invalid"}
return {
"valid": True,
"proposal_id": proposal_id,
"status": status,
"applied": applied,
}
return {"valid": False, "error": "reply did not contain a KB_STATE line"}
def reply_state_receipt_matches(receipt: dict[str, Any], proposal: dict[str, Any] | None) -> bool:
if not receipt.get("valid") or not proposal:
return False
expected_applied = bool(proposal.get("applied_at")) and proposal.get("status") == "applied"
return (
receipt.get("proposal_id") == str(proposal.get("id"))
and receipt.get("status") == proposal.get("status")
and receipt.get("applied") is expected_applied
)
def write_report(path: Path, report: dict[str, Any], *, secret_values: tuple[str, ...] = ()) -> None:
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
directory_fd = os.open(path.parent, directory_flags)
temporary_name = f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
try:
payload = (
json.dumps(redact_value(report, secret_values=secret_values), indent=2, sort_keys=True) + "\n"
).encode()
descriptor = os.open(
temporary_name,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
0o600,
dir_fd=directory_fd,
)
try:
with os.fdopen(descriptor, "wb") as handle:
handle.write(payload)
handle.flush()
os.fsync(handle.fileno())
except BaseException:
try:
os.close(descriptor)
except OSError:
pass
raise
os.replace(temporary_name, path.name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd)
os.chmod(path.name, 0o600, dir_fd=directory_fd, follow_symlinks=False)
finally:
try:
os.unlink(temporary_name, dir_fd=directory_fd)
except OSError:
pass
os.close(directory_fd)
async def run_checkpoint(args: argparse.Namespace) -> dict[str, Any]:
global _LAST_TERMINATION_SIGNAL
_LAST_TERMINATION_SIGNAL = None
validate_args(args)
report: dict[str, Any] = {
"schema": "livingip.leoCloneBoundHandlerCheckpoint.v1",
"generated_at_utc": utc_now(),
"mode": "live_vps_gatewayrunner_disposable_db_no_send",
"required_tier": "T2_live_vps_no_send_handler_bound_to_disposable_db",
"posted_to_telegram": False,
"orchestrates_review_or_apply": False,
"changes_live_profile": False,
"restarts_service": False,
"production_read_only_invariant_queries": True,
"target": {"container": args.container, "database": args.db},
"prompt": {
"id": args.prompt_id,
"text": args.prompt,
"expected_state": args.expected_state,
"proposal_id": args.proposal_id,
},
"source": {
"platform": "telegram",
"chat_id": args.chat_id,
"chat_name": "Leo",
"chat_type": "group",
"user_id": args.user_id,
"user_name": args.user_name,
},
"output": str(args.output),
"errors": [],
}
temp_profile: Path | None = None
tool_log: Path | None = None
production_service_before: dict[str, Any] | None = None
production_counts_before: dict[str, int] | None = None
production_guard_before: dict[str, Any] | None = None
target_before: dict[str, Any] | None = None
target_after: dict[str, Any] | None = None
target_guard_before: dict[str, Any] | None = None
target_guard_after: dict[str, Any] | None = None
target_identity_before: dict[str, Any] | None = None
target_identity_after: dict[str, Any] | None = None
target_container_id: str | None = None
target_read_only_state: dict[str, Any] | None = None
target_read_only_restore_required = False
target_read_only_restored: bool | None = None
gateway: dict[str, Any] = {}
selection: dict[str, Any] = {}
bridge: dict[str, Any] = {}
live_hashes_before: dict[str, str | None] | None = None
provider_environment: dict[str, str] = {}
provider_secret_values: tuple[str, ...] = ()
try:
production_service_before = service_state()
production_guard_before = database_guard_snapshot(PRODUCTION_CONTAINER, PRODUCTION_DB)
production_counts_before = production_guard_before["counts"]
live_hashes_before = bridge_hashes()
target_identity_before = container_identity(args.container)
production_identity = container_identity(PRODUCTION_CONTAINER)
isolation_checks = validate_disposable_target_identity(target_identity_before, production_identity)
target_container_id = str(target_identity_before["id"])
report["target"].update(
{
"requested_container": args.container,
"bound_container_id": target_container_id,
"container_identity_before": target_identity_before,
"isolation_checks": isolation_checks,
}
)
report["production_invariants"] = {
"container_identity": production_identity,
"counts_before": production_counts_before,
"guard_snapshot_before": production_guard_before,
"service_before": production_service_before,
}
target_read_only_prior = target_read_only_setting(target_container_id, args.db)
target_read_only_restore_required = not bool(target_read_only_prior.get("already_forced_read_only"))
report["target"]["read_only_role_restore_obligation_installed"] = target_read_only_restore_required
target_read_only_state = configure_target_read_only(target_container_id, args.db, enabled=True)
report["target"]["read_only_role_binding"] = target_read_only_state
target_guard_before = database_guard_snapshot(target_container_id, args.db)
if target_guard_before.get("database_identity", {}).get("system_identifier") == production_guard_before.get(
"database_identity", {}
).get("system_identifier"):
raise CheckpointError("target and production resolve to the same Postgres system identifier")
target_before = target_checkpoint(target_container_id, args.db)
if target_before.get("database_identity", {}).get("current_database") != args.db:
raise CheckpointError("target checkpoint did not read the explicitly supplied database")
report["target_checkpoint_before"] = target_checkpoint_receipt_view(target_before)
report["target_invariants"] = {"guard_snapshot_before": target_guard_before}
temp_profile, copy_audit = copy_profile(run_id=args.run_id, prompt_id=args.prompt_id)
report["temporary_profile_copy_audit"] = copy_audit
if args.copy_model_auth:
report["model_auth_binding"] = copy_ephemeral_model_auth(temp_profile)
provider_environment, environment_audit = load_ephemeral_provider_environment(temp_profile)
provider_secret_values = tuple(provider_environment.values())
report["model_auth_binding"]["environment_binding"] = environment_audit
else:
report["model_auth_binding"] = {
"mode": "excluded",
"source_contents_recorded": False,
"source_fingerprint_recorded": False,
}
bridge = patch_temp_bridge(temp_profile, target_container_id, args.db)
tool_log = Path(bridge["tool_log_path"])
report["temporary_bridge"] = bridge
gateway = await invoke_gateway_subprocess(
args,
temp_profile,
provider_environment=provider_environment,
)
report["gateway"] = gateway
if gateway.get("handler_error"):
report["errors"].append(
{
"type": "GatewayHandlerError",
"message": str(gateway["handler_error"]),
}
)
target_after = target_checkpoint(target_container_id, args.db)
target_guard_after = database_guard_snapshot(target_container_id, args.db)
target_identity_after = container_identity(args.container)
if target_identity_after.get("id") != target_container_id:
raise CheckpointError("target container name no longer resolves to the bound container ID")
report["target"]["container_identity_after"] = target_identity_after
report["target_checkpoint_after"] = target_checkpoint_receipt_view(target_after)
selection = select_expected_proposal(
before=target_before,
after=target_after,
expected_state=args.expected_state,
prompt=args.prompt,
reply=str(gateway.get("reply", "")),
explicit_proposal_id=args.proposal_id,
)
selected = selection.get("selected_proposal")
identifiers = _uuid_values(selected) if selected else set()
related_rows = related_canonical_rows(target_container_id, args.db, identifiers)
report["proposal_checkpoint"] = {
**selection,
"selected_proposal": proposal_receipt_view(selected) if selected else None,
"payload_uuid_values": sorted(identifiers),
"related_canonical_rows": canonical_rows_receipt_view(related_rows),
}
except Exception as exc:
report["errors"].append({"type": type(exc).__name__, "message": str(exc)})
report["traceback_tail"] = traceback.format_exc().splitlines()[-12:]
finally:
provider_environment.clear()
if target_before is not None and target_after is None and target_container_id:
try:
target_after = target_checkpoint(target_container_id, args.db)
report["target_checkpoint_after"] = target_checkpoint_receipt_view(target_after)
except Exception as exc:
report["errors"].append({"type": type(exc).__name__, "message": f"target postflight: {exc}"})
if target_guard_before is not None and target_guard_after is None and target_container_id:
try:
target_guard_after = database_guard_snapshot(target_container_id, args.db)
except Exception as exc:
report["errors"].append({"type": type(exc).__name__, "message": f"target guard postflight: {exc}"})
if target_identity_before is not None and target_identity_after is None:
try:
target_identity_after = container_identity(args.container)
report["target"]["container_identity_after"] = target_identity_after
except Exception as exc:
report["errors"].append({"type": type(exc).__name__, "message": f"target identity postflight: {exc}"})
if target_container_id and (target_read_only_state is not None or target_read_only_restore_required):
try:
if target_read_only_restore_required:
reset = configure_target_read_only(target_container_id, args.db, enabled=False)
target_read_only_restored = reset.get("verified_transaction_read_only") == "off"
else:
target_read_only_restored = True
except Exception as exc:
target_read_only_restored = False
report["errors"].append({"type": type(exc).__name__, "message": f"target read-only reset: {exc}"})
report.setdefault("target", {})["read_only_role_restored"] = target_read_only_restored
try:
if not bridge or not target_container_id or not target_before:
raise CheckpointError("tool proof prerequisites were not established")
tool_proof = read_tool_proof(
tool_log,
target_container_id,
args.db,
run_nonce=str(bridge["run_nonce"]),
database_identity=target_before["database_identity"],
)
except Exception as exc:
report["errors"].append({"type": type(exc).__name__, "message": f"tool proof readback: {exc}"})
tool_proof = {
"parse_errors": [str(exc)],
"event_count": 0,
"duplicate_invocation_ids": [],
"complete_start_end_pairing": False,
"invocations": [],
"invocation_count": 0,
"all_bound_to_supplied_target": False,
"all_completed_successfully": False,
}
report["tool_proof"] = tool_proof
temp_root = temp_profile.parent if temp_profile else None
child_process = gateway.get("child_process") or {}
child_absent = not _ACTIVE_GATEWAY_CHILDREN and (
not child_process
or (
child_process.get("alive_after_readback") is False
and child_process.get("process_group_alive_after_readback") is False
)
)
if child_absent:
temp_removed = remove_tree(temp_root)
else:
temp_removed = False
report["errors"].append(
{"type": "CleanupSafetyError", "message": "temporary profile retained because gateway group is alive"}
)
absence_verified = temp_path_absent(temp_root)
report["cleanup"] = {
"temp_root": str(temp_root) if temp_root else None,
"temp_profile_removed": temp_removed,
"temp_session_removed_with_profile": temp_removed,
"absence_verified_independently": absence_verified,
"active_temp_root_registry_clear": temp_root not in _ACTIVE_TEMP_ROOTS if temp_root else True,
"termination_signal": _LAST_TERMINATION_SIGNAL,
"gateway_group_confirmed_absent_before_profile_removal": child_absent,
}
try:
production_guard_after = database_guard_snapshot(PRODUCTION_CONTAINER, PRODUCTION_DB)
production_counts_after = production_guard_after["counts"]
except Exception as exc:
production_guard_after = None
production_counts_after = None
report["errors"].append({"type": type(exc).__name__, "message": f"production count postflight: {exc}"})
try:
production_service_after = service_state()
except Exception as exc: # service_state currently returns errors instead of raising
production_service_after = None
report["errors"].append({"type": type(exc).__name__, "message": f"service postflight: {exc}"})
live_hashes_after = bridge_hashes()
production = report.setdefault("production_invariants", {})
production.update(
{
"counts_before": production_counts_before,
"counts_after": production_counts_after,
"counts_unchanged": production_counts_before is not None
and production_counts_before == production_counts_after,
"guard_snapshot_before": production_guard_before,
"guard_snapshot_after": production_guard_after,
"guard_snapshot_unchanged": production_guard_before is not None
and production_guard_before == production_guard_after,
"service_before": production_service_before,
"service_after": production_service_after,
"service_unchanged": _same_service(production_service_before, production_service_after),
"live_bridge_hashes_before": live_hashes_before,
"live_bridge_hashes_after": live_hashes_after,
"live_bridge_unchanged": live_hashes_before is not None and live_hashes_before == live_hashes_after,
}
)
target_invariants = report.setdefault("target_invariants", {})
target_invariants.update(
{
"guard_snapshot_before": target_guard_before,
"guard_snapshot_after": target_guard_after,
"guard_snapshot_unchanged": target_guard_before is not None
and target_guard_before == target_guard_after,
"container_identity_before": target_identity_before,
"container_identity_after": target_identity_after,
"container_identity_unchanged": target_identity_before is not None
and target_identity_before == target_identity_after,
"postgres_system_distinct_from_production": bool(target_guard_before and production_guard_before)
and target_guard_before.get("database_identity", {}).get("system_identifier")
!= production_guard_before.get("database_identity", {}).get("system_identifier"),
"read_only_during_checkpoint": bool(target_guard_before and target_guard_after)
and target_guard_before.get("database_identity", {}).get("transaction_read_only") == "on"
and target_guard_after.get("database_identity", {}).get("transaction_read_only") == "on",
"read_only_role_restored": target_read_only_restored is True,
}
)
selected = selection.get("selected_proposal") if selection else None
reply = str(gateway.get("reply", ""))
tool_surface = gateway.get("tool_surface") or {}
allowed_tool_names = set(tool_surface.get("allowed_tools") or [])
transcript_calls = [
event.get("tool_name")
for event in (gateway.get("transcript_tool_trace") or {}).get("events", [])
if event.get("phase") == "call"
]
terminal_attempts = gateway.get("transcript_terminal_attempts") or transcript_terminal_attempt_summary(
(gateway.get("transcript_tool_trace") or {}).get("events", [])
)
gateway["transcript_terminal_attempts"] = terminal_attempts
child_process = gateway.get("child_process") or {}
reply_state_receipt = parse_reply_state_receipt(reply)
gateway["reply_state_receipt"] = reply_state_receipt
checks = {
"target_is_not_production_container": target_invariants.get("postgres_system_distinct_from_production")
is True,
"target_container_identity_unchanged": target_invariants.get("container_identity_unchanged") is True,
"target_isolation_contract_passed": bool(report.get("target", {}).get("isolation_checks"))
and all(report.get("target", {}).get("isolation_checks", {}).values()),
"target_database_matches_supplied_db": bool(target_after)
and target_after.get("database_identity", {}).get("current_database") == args.db,
"target_row_content_unchanged": target_invariants.get("guard_snapshot_unchanged") is True,
"target_forced_read_only_during_checkpoint": target_invariants.get("read_only_during_checkpoint") is True,
"target_read_only_role_restored": target_invariants.get("read_only_role_restored") is True,
"telegram_shaped_source_authorized": gateway.get("authorized") is True,
"real_gateway_handler_invoked": gateway.get("handler_invoked") is True,
"machine_readable_reply_nonempty": bool(reply.strip()),
"model_free_fallback_not_used": gateway.get("model_free_fallback_used") is False,
"kb_tool_call_observed": tool_proof["invocation_count"] > 0,
"all_kb_tool_calls_bound_to_supplied_target": tool_proof["all_bound_to_supplied_target"],
"all_kb_tool_calls_completed_successfully": tool_proof["all_completed_successfully"],
"selected_exactly_one_expected_proposal": selection.get("selected_exactly_one") is True,
"expected_state_semantics_match": selection.get("state_semantics", {}).get("all") is True,
"reply_mentions_selected_proposal": _reply_mentions_proposal(reply, selected),
"tool_receipt_read_exact_selected_proposal": tool_proof_read_exact_proposal(tool_proof, selected),
"reply_state_receipt_valid": reply_state_receipt.get("valid") is True,
"reply_state_receipt_matches_selected_row": reply_state_receipt_matches(reply_state_receipt, selected),
"production_counts_unchanged": production.get("counts_unchanged") is True,
"production_guard_snapshot_unchanged": production.get("guard_snapshot_unchanged") is True,
"gateway_service_unchanged": production.get("service_unchanged") is True,
"live_bridge_unchanged": production.get("live_bridge_unchanged") is True,
"send_message_tool_not_exposed": tool_surface.get("send_message_tool_enabled") is False,
"actual_registry_tools_exactly_allowlisted": allowed_tool_names == {"skills_list", "skill_view", "terminal"}
and set(tool_surface.get("actual_registry_tools") or []) == allowed_tool_names,
"gateway_has_no_delivery_adapters": tool_surface.get("gateway_adapters_verified_mapping") is True
and tool_surface.get("gateway_adapter_count") == 0,
"terminal_restricted_to_clone_wrapper": (tool_surface.get("terminal_restricted_to_clone_wrapper") is True),
"terminal_subprocess_has_no_provider_credentials": (
tool_surface.get("terminal_subprocess_inherits_provider_credentials") is False
),
"terminal_command_rejections_absent": terminal_attempts.get("rejected_call_count") == 0,
"terminal_nonzero_results_absent": terminal_attempts.get("nonzero_call_count") == 0,
"terminal_call_results_complete_and_receipted": terminal_attempts.get("result_code_unknown_call_count") == 0
and terminal_attempts.get("call_count") == tool_proof.get("invocation_count"),
"all_transcript_tool_calls_within_allowlist": bool(transcript_calls)
and all(name in allowed_tool_names for name in transcript_calls),
"gateway_child_and_descendants_absent": (
child_process.get("alive_after_readback") is False
and child_process.get("process_group_alive_after_readback") is False
),
"gateway_child_ready_in_own_process_group": child_process.get("readiness_verified") is True,
"gateway_child_completed_without_timeout_or_forced_kill": child_process.get("timed_out") is False
and child_process.get("exitcode") == 0
and (child_process.get("termination") or {}).get("attempted") is False,
"gateway_result_used_private_file": child_process.get("result_transport") == "private_temp_file",
"temp_profile_removed": temp_removed,
"temp_session_removed": temp_removed,
"temp_root_absence_verified_independently": absence_verified,
"temp_root_unregistered": report["cleanup"]["active_temp_root_registry_clear"],
"posted_to_telegram_false": (
report["posted_to_telegram"] is False
and gateway.get("posted_to_telegram") is False
and tool_surface.get("send_message_tool_enabled") is False
and tool_surface.get("gateway_adapter_count") == 0
),
"review_apply_not_orchestrated": report["orchestrates_review_or_apply"] is False
and target_invariants.get("guard_snapshot_unchanged") is True,
}
report["checks"] = checks
report["status"] = "pass" if not report["errors"] and all(checks.values()) else "fail"
report["claim_ceiling"] = (
"T2 live VPS GatewayRunner checkpoint with a KB-only tool allowlist, a terminal handler restricted "
"to read-only commands on the supplied disposable Postgres target, no delivery adapters, and no "
"send_message tool. It reads production invariants before/after but does not review, approve, apply, "
"restart, post, or expose a production mutation tool to the model."
)
report["completed_at_utc"] = utc_now()
report = redact_value(report, secret_values=provider_secret_values)
write_report(args.output, report, secret_values=provider_secret_values)
return report
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--container", required=True, help="Disposable Postgres Docker container name.")
parser.add_argument("--db", required=True, help="Database inside the disposable container.")
parser.add_argument("--prompt-id", required=True, help="Stable checkpoint prompt identifier.")
parser.add_argument("--prompt", required=True, help="Exact operator prompt passed to GatewayRunner.")
parser.add_argument("--expected-state", required=True, choices=EXPECTED_STATES)
parser.add_argument("--output", required=True, type=Path, help="Machine-readable JSON receipt path.")
parser.add_argument(
"--proposal-id",
help="Optional exact proposal UUID. Otherwise selection uses prompt/reply UUIDs or an unambiguous state delta.",
)
parser.add_argument("--chat-id", default=DEFAULT_CHAT_ID)
parser.add_argument("--user-id", default=DEFAULT_USER_ID)
parser.add_argument("--user-name", default=DEFAULT_USER_NAME)
parser.add_argument("--run-id", default=datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S"))
parser.add_argument("--turn-timeout", type=int, default=300)
parser.add_argument(
"--copy-model-auth",
action="store_true",
help=(
"Copy only the live profile auth.json into the UUID-scoped temporary "
"profile. The copy may refresh independently and is deleted with the "
"temporary profile; credential contents are never retained in the receipt."
),
)
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
try:
with termination_cleanup_handlers():
report = asyncio.run(run_checkpoint(args))
except CheckpointError as exc:
print(json.dumps({"status": "rejected", "error": str(exc)}, sort_keys=True))
return 2
except TerminationRequested as exc:
print(json.dumps({"status": "terminated", "signal": exc.signum, "output": str(args.output)}, sort_keys=True))
return 128 + exc.signum
print(
json.dumps(
{
"status": report["status"],
"output": str(args.output),
"checks_passed": sum(1 for value in report.get("checks", {}).values() if value is True),
"checks_total": len(report.get("checks", {})),
"error_count": len(report.get("errors", [])),
},
sort_keys=True,
)
)
return 0 if report["status"] == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main())