2374 lines
97 KiB
Python
2374 lines
97 KiB
Python
#!/usr/bin/env python3
|
|
"""Run one no-send Leo GatewayRunner checkpoint against a disposable Postgres target.
|
|
|
|
Run this command on the Leo VPS as the ``teleo`` user while the disposable
|
|
container/database exists. The command copies the live profile, detaches and
|
|
patches only the temporary Teleo KB bridge, invokes the real GatewayRunner
|
|
handler, and removes the temporary profile and session before returning.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import hashlib
|
|
import inspect
|
|
import json
|
|
import multiprocessing
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import traceback
|
|
import uuid
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import urlsplit
|
|
|
|
LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
|
|
AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent")
|
|
PRODUCTION_CONTAINER = "teleo-pg"
|
|
PRODUCTION_DB = "teleo"
|
|
GATEWAY_SERVICE = "leoclean-gateway.service"
|
|
DISPOSABLE_LABEL_KEY = "com.livingip.leo.checkpoint"
|
|
DISPOSABLE_LABEL_VALUE = "disposable"
|
|
SYSTEM_EXEC_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
SYSTEM_DOCKER = "/usr/bin/docker"
|
|
|
|
DEFAULT_CHAT_ID = "-5146042086"
|
|
DEFAULT_USER_ID = "9070919"
|
|
DEFAULT_USER_NAME = "codex clone checkpoint"
|
|
|
|
EXPECTED_STATES = ("pending_review", "approved", "applied")
|
|
READ_ONLY_BRIDGE_COMMANDS = frozenset(
|
|
{
|
|
"context",
|
|
"search",
|
|
"show",
|
|
"evidence",
|
|
"edges",
|
|
"list-proposals",
|
|
"search-proposals",
|
|
"show-proposal",
|
|
"decision-matrix-status",
|
|
}
|
|
)
|
|
HARMLESS_TRAILING_REDIRECT_TOKENS = frozenset({"2>&1"})
|
|
CANONICAL_TABLES = (
|
|
"public.claims",
|
|
"public.sources",
|
|
"public.claim_evidence",
|
|
"public.claim_edges",
|
|
"public.reasoning_tools",
|
|
)
|
|
COUNT_TABLES = (*CANONICAL_TABLES, "kb_stage.kb_proposals")
|
|
UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b")
|
|
SAFE_DOCKER_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*\Z")
|
|
SAFE_DB_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_$.-]*\Z")
|
|
SAFE_ENV_NAME_RE = re.compile(r"[A-Z][A-Z0-9_]*\Z")
|
|
SECRET_PATTERNS = (
|
|
re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{20,}\b"),
|
|
re.compile(r"\bsk-(?:or-v1-)?[A-Za-z0-9_-]{16,}\b"),
|
|
re.compile(r"\beyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{10,}(?:\.[A-Za-z0-9_-]{10,})?\b"),
|
|
re.compile(r"(Authorization:\s*Bearer\s+)([A-Za-z0-9._-]+)", re.IGNORECASE),
|
|
re.compile(r"((?:api[_-]?key|token|secret|password)\s*[=:]\s*)([^\s,;]+)", re.IGNORECASE),
|
|
re.compile(
|
|
r"((?:access_token|refresh_token|api_key|authorization|cookie)[\"']?\s*:\s*[\"'])([^\"']+)",
|
|
re.IGNORECASE,
|
|
),
|
|
)
|
|
SENSITIVE_KEY_RE = re.compile(
|
|
r"(?:^|_)(?:access_token|refresh_token|api_key|authorization|bot_token|client_secret|cookie|credential|key|"
|
|
r"password|private_key|secret|signing_key|token)(?:$|_)",
|
|
re.IGNORECASE,
|
|
)
|
|
_ACTIVE_TEMP_ROOTS: set[Path] = set()
|
|
_ACTIVE_GATEWAY_CHILDREN: set[Any] = set()
|
|
_LAST_TERMINATION_SIGNAL: int | None = None
|
|
|
|
|
|
class CheckpointError(RuntimeError):
|
|
"""Raised when a checkpoint cannot produce trustworthy proof."""
|
|
|
|
|
|
class TerminationRequested(BaseException):
|
|
"""Turn an interruptible termination signal into normal ``finally`` cleanup."""
|
|
|
|
def __init__(self, signum: int) -> None:
|
|
super().__init__(f"termination signal {signum}")
|
|
self.signum = signum
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def sha256_bytes(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def json_sha256(value: Any) -> str:
|
|
encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()
|
|
return sha256_bytes(encoded)
|
|
|
|
|
|
def redact_text(value: str, *, secret_values: tuple[str, ...] = ()) -> str:
|
|
result = value
|
|
for secret in sorted({item for item in secret_values if len(item) >= 4}, key=len, reverse=True):
|
|
result = result.replace(secret, "[REDACTED_SECRET]")
|
|
for pattern in SECRET_PATTERNS:
|
|
if pattern.groups >= 2:
|
|
result = pattern.sub(r"\1[REDACTED]", result)
|
|
else:
|
|
result = pattern.sub("[REDACTED_TOKEN]", result)
|
|
return result
|
|
|
|
|
|
def redact_value(value: Any, *, key: str | None = None, secret_values: tuple[str, ...] = ()) -> Any:
|
|
if key and SENSITIVE_KEY_RE.search(key) and isinstance(value, (str, bytes, dict, list, tuple)):
|
|
return "[REDACTED]"
|
|
if isinstance(value, dict):
|
|
return {
|
|
item_key: redact_value(item, key=str(item_key), secret_values=secret_values)
|
|
for item_key, item in value.items()
|
|
}
|
|
if isinstance(value, list):
|
|
return [redact_value(item, secret_values=secret_values) for item in value]
|
|
if isinstance(value, tuple):
|
|
return [redact_value(item, secret_values=secret_values) for item in value]
|
|
if isinstance(value, str):
|
|
return redact_text(value, secret_values=secret_values)
|
|
return value
|
|
|
|
|
|
def run_command(command: list[str], *, input_text: str | None = None) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
command,
|
|
input=input_text,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def _command_failure(command: list[str], proc: subprocess.CompletedProcess[str]) -> CheckpointError:
|
|
detail = redact_text((proc.stderr or proc.stdout).strip())
|
|
return CheckpointError(f"command failed ({proc.returncode}): {shlex.join(command)}: {detail}")
|
|
|
|
|
|
def service_state() -> dict[str, Any]:
|
|
command = [
|
|
"systemctl",
|
|
"show",
|
|
GATEWAY_SERVICE,
|
|
"-p",
|
|
"ActiveState",
|
|
"-p",
|
|
"SubState",
|
|
"-p",
|
|
"MainPID",
|
|
"-p",
|
|
"NRestarts",
|
|
"-p",
|
|
"ExecMainStartTimestamp",
|
|
"-p",
|
|
"User",
|
|
"-p",
|
|
"WorkingDirectory",
|
|
"--no-pager",
|
|
]
|
|
proc = run_command(command)
|
|
state: dict[str, Any] = {"returncode": proc.returncode}
|
|
for line in proc.stdout.splitlines():
|
|
key, separator, value = line.partition("=")
|
|
if separator:
|
|
state[key] = value
|
|
if proc.returncode != 0:
|
|
state["error"] = (proc.stderr or proc.stdout).strip()
|
|
return state
|
|
|
|
|
|
def container_identity(container: str) -> dict[str, Any]:
|
|
command = ["docker", "inspect", container]
|
|
proc = run_command(command)
|
|
if proc.returncode != 0:
|
|
raise _command_failure(command, proc)
|
|
try:
|
|
payload = json.loads(proc.stdout)
|
|
item = payload[0]
|
|
except (json.JSONDecodeError, IndexError, KeyError, TypeError) as exc:
|
|
raise CheckpointError(f"docker inspect returned invalid JSON for {container!r}") from exc
|
|
identity = str(item.get("Id") or "")
|
|
if not identity:
|
|
raise CheckpointError(f"docker inspect returned an invalid identity for {container!r}")
|
|
config = item.get("Config") if isinstance(item.get("Config"), dict) else {}
|
|
host_config = item.get("HostConfig") if isinstance(item.get("HostConfig"), dict) else {}
|
|
network = item.get("NetworkSettings") if isinstance(item.get("NetworkSettings"), dict) else {}
|
|
mounts = item.get("Mounts") if isinstance(item.get("Mounts"), list) else []
|
|
return {
|
|
"id": identity,
|
|
"name": str(item.get("Name") or "").lstrip("/"),
|
|
"image": str(config.get("Image") or ""),
|
|
"labels": dict(config.get("Labels") or {}),
|
|
"network_mode": host_config.get("NetworkMode"),
|
|
"ports": network.get("Ports") or {},
|
|
"mount_sources": sorted(
|
|
str(mount.get("Source")) for mount in mounts if isinstance(mount, dict) and mount.get("Source")
|
|
),
|
|
"started_at": (item.get("State") or {}).get("StartedAt") if isinstance(item.get("State"), dict) else None,
|
|
}
|
|
|
|
|
|
def validate_disposable_target_identity(target: dict[str, Any], production: dict[str, Any]) -> dict[str, bool]:
|
|
target_mounts = set(target.get("mount_sources") or [])
|
|
production_mounts = set(production.get("mount_sources") or [])
|
|
checks = {
|
|
"different_container_id": bool(target.get("id")) and target.get("id") != production.get("id"),
|
|
"disposable_label_present": (target.get("labels") or {}).get(DISPOSABLE_LABEL_KEY) == DISPOSABLE_LABEL_VALUE,
|
|
"network_disabled": target.get("network_mode") == "none",
|
|
"no_published_ports": not bool(target.get("ports")),
|
|
"no_shared_mount_sources": not bool(target_mounts & production_mounts),
|
|
}
|
|
if not all(checks.values()):
|
|
failed = sorted(name for name, value in checks.items() if not value)
|
|
raise CheckpointError(f"supplied target is not a proven disposable isolated container: {failed}")
|
|
return checks
|
|
|
|
|
|
def _psql_json(container: str, database: str, sql: str) -> Any:
|
|
command = [
|
|
"docker",
|
|
"exec",
|
|
"-i",
|
|
container,
|
|
"psql",
|
|
"-U",
|
|
"postgres",
|
|
"-d",
|
|
database,
|
|
"-At",
|
|
"-q",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
]
|
|
proc = run_command(command, input_text=sql)
|
|
if proc.returncode != 0:
|
|
raise _command_failure(command, proc)
|
|
raw = proc.stdout.strip()
|
|
if not raw:
|
|
raise CheckpointError(f"psql returned no JSON for {container}/{database}")
|
|
try:
|
|
return json.loads(raw.splitlines()[-1])
|
|
except json.JSONDecodeError as exc:
|
|
raise CheckpointError(f"psql returned invalid JSON for {container}/{database}: {exc}") from exc
|
|
|
|
|
|
def _sql_identifier(value: str) -> str:
|
|
return '"' + value.replace('"', '""') + '"'
|
|
|
|
|
|
def target_read_only_setting(container: str, database: str) -> dict[str, Any]:
|
|
return _psql_json(
|
|
container,
|
|
database,
|
|
r"""\set ON_ERROR_STOP on
|
|
select jsonb_build_object(
|
|
'already_forced_read_only', exists (
|
|
select 1
|
|
from pg_db_role_setting settings
|
|
join pg_roles role_row on role_row.oid = settings.setrole
|
|
join pg_database database_row on database_row.oid = settings.setdatabase
|
|
where role_row.rolname = 'postgres'
|
|
and database_row.datname = current_database()
|
|
and 'default_transaction_read_only=on' = any(settings.setconfig)
|
|
)
|
|
)::text;
|
|
""",
|
|
)
|
|
|
|
|
|
def configure_target_read_only(container: str, database: str, *, enabled: bool) -> dict[str, Any]:
|
|
database_identifier = _sql_identifier(database)
|
|
if enabled:
|
|
setting = target_read_only_setting(container, database)
|
|
command_sql = f"alter role postgres in database {database_identifier} set default_transaction_read_only = on;"
|
|
else:
|
|
setting = {"already_forced_read_only": False}
|
|
command_sql = (
|
|
"set default_transaction_read_only = off; "
|
|
f"alter role postgres in database {database_identifier} reset default_transaction_read_only;"
|
|
)
|
|
command = [
|
|
"docker",
|
|
"exec",
|
|
"-i",
|
|
container,
|
|
"psql",
|
|
"-U",
|
|
"postgres",
|
|
"-d",
|
|
database,
|
|
"-At",
|
|
"-q",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
]
|
|
proc = run_command(command, input_text=command_sql)
|
|
if proc.returncode != 0:
|
|
raise _command_failure(command, proc)
|
|
verification = _psql_json(
|
|
container,
|
|
database,
|
|
"select jsonb_build_object('transaction_read_only', current_setting('transaction_read_only'))::text;",
|
|
)
|
|
expected = "on" if enabled else "off"
|
|
if verification.get("transaction_read_only") != expected:
|
|
raise CheckpointError(f"target read-only verification expected {expected!r}: {verification}")
|
|
return {
|
|
"enabled": enabled,
|
|
"already_forced_read_only": bool(setting.get("already_forced_read_only")),
|
|
"verified_transaction_read_only": verification.get("transaction_read_only"),
|
|
}
|
|
|
|
|
|
def database_counts(
|
|
container: str,
|
|
database: str,
|
|
*,
|
|
tables: tuple[str, ...] = COUNT_TABLES,
|
|
) -> dict[str, int]:
|
|
pairs = ",\n ".join(f"'{table}', (select count(*) from {table})" for table in tables)
|
|
result = _psql_json(
|
|
container,
|
|
database,
|
|
f"""\\set ON_ERROR_STOP on
|
|
begin transaction read only;
|
|
select jsonb_build_object(
|
|
{pairs}
|
|
)::text;
|
|
rollback;
|
|
""",
|
|
)
|
|
return {str(key): int(value) for key, value in result.items()}
|
|
|
|
|
|
def database_guard_snapshot(
|
|
container: str,
|
|
database: str,
|
|
*,
|
|
tables: tuple[str, ...] = COUNT_TABLES,
|
|
) -> dict[str, Any]:
|
|
count_pairs = ",\n ".join(f"'{table}', (select count(*) from {table})" for table in tables)
|
|
hash_pairs = ",\n ".join(
|
|
f"'{table}', (select md5(coalesce(string_agg(to_jsonb(t)::text, E'\\n' "
|
|
f"order by to_jsonb(t)::text), '')) from {table} t)"
|
|
for table in tables
|
|
)
|
|
result = _psql_json(
|
|
container,
|
|
database,
|
|
rf"""\set ON_ERROR_STOP on
|
|
begin transaction read only;
|
|
select jsonb_build_object(
|
|
'database_identity', jsonb_build_object(
|
|
'current_database', current_database(),
|
|
'system_identifier', (select system_identifier::text from pg_control_system()),
|
|
'postmaster_started_at', pg_postmaster_start_time()::text,
|
|
'transaction_read_only', current_setting('transaction_read_only')
|
|
),
|
|
'counts', jsonb_build_object(
|
|
{count_pairs}
|
|
),
|
|
'rowset_md5', jsonb_build_object(
|
|
{hash_pairs}
|
|
)
|
|
)::text;
|
|
rollback;
|
|
""",
|
|
)
|
|
return {
|
|
"database_identity": {str(key): value for key, value in result["database_identity"].items()},
|
|
"counts": {str(key): int(value) for key, value in result["counts"].items()},
|
|
"rowset_md5": {str(key): str(value) for key, value in result["rowset_md5"].items()},
|
|
}
|
|
|
|
|
|
def target_checkpoint(container: str, database: str) -> dict[str, Any]:
|
|
pairs = ",\n ".join(f"'{table}', (select count(*) from {table})" for table in COUNT_TABLES)
|
|
checkpoint = _psql_json(
|
|
container,
|
|
database,
|
|
f"""\\set ON_ERROR_STOP on
|
|
begin transaction read only;
|
|
select jsonb_build_object(
|
|
'database_identity', jsonb_build_object(
|
|
'current_database', current_database(),
|
|
'database_oid', (select oid::text from pg_database where datname = current_database()),
|
|
'system_identifier', (select system_identifier::text from pg_control_system()),
|
|
'server_address', coalesce(inet_server_addr()::text, 'local_socket'),
|
|
'server_port', inet_server_port(),
|
|
'postmaster_started_at', pg_postmaster_start_time()::text,
|
|
'transaction_read_only', current_setting('transaction_read_only'),
|
|
'transaction_snapshot', txid_current_snapshot()::text
|
|
),
|
|
'counts', jsonb_build_object(
|
|
{pairs}
|
|
),
|
|
'proposals', coalesce(
|
|
(select jsonb_agg(to_jsonb(p) order by p.id::text) from kb_stage.kb_proposals p),
|
|
'[]'::jsonb
|
|
)
|
|
)::text;
|
|
rollback;
|
|
""",
|
|
)
|
|
checkpoint["sha256"] = json_sha256(checkpoint)
|
|
return checkpoint
|
|
|
|
|
|
def proposal_receipt_view(proposal: dict[str, Any]) -> dict[str, Any]:
|
|
payload = proposal.get("payload")
|
|
return {
|
|
"id": proposal.get("id"),
|
|
"proposal_type": proposal.get("proposal_type"),
|
|
"status": proposal.get("status"),
|
|
"proposed_by": proposal.get("proposed_by"),
|
|
"channel": proposal.get("channel"),
|
|
"source_ref": proposal.get("source_ref"),
|
|
"created_at": proposal.get("created_at"),
|
|
"reviewed_at": proposal.get("reviewed_at"),
|
|
"reviewed_by_handle": proposal.get("reviewed_by_handle"),
|
|
"applied_at": proposal.get("applied_at"),
|
|
"applied_by_handle": proposal.get("applied_by_handle"),
|
|
"payload_sha256": json_sha256(payload),
|
|
"payload_uuid_values": sorted(_uuid_values(payload)),
|
|
}
|
|
|
|
|
|
def target_checkpoint_receipt_view(checkpoint: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"database_identity": checkpoint.get("database_identity"),
|
|
"counts": checkpoint.get("counts"),
|
|
"proposals": [proposal_receipt_view(item) for item in checkpoint.get("proposals", [])],
|
|
"sha256": checkpoint.get("sha256"),
|
|
}
|
|
|
|
|
|
def canonical_rows_receipt_view(rows_by_table: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
|
|
identity_fields = {
|
|
"id",
|
|
"claim_id",
|
|
"source_id",
|
|
"from_claim",
|
|
"to_claim",
|
|
"reasoning_tool_id",
|
|
"slug",
|
|
"key",
|
|
"type",
|
|
"edge_type",
|
|
}
|
|
result: dict[str, list[dict[str, Any]]] = {}
|
|
for table, rows in rows_by_table.items():
|
|
result[table] = [
|
|
{
|
|
**{key: value for key, value in row.items() if key in identity_fields},
|
|
"row_sha256": json_sha256(row),
|
|
}
|
|
for row in rows
|
|
if isinstance(row, dict)
|
|
]
|
|
return result
|
|
|
|
|
|
def _sql_text(value: str) -> str:
|
|
return "'" + value.replace("'", "''") + "'"
|
|
|
|
|
|
def related_canonical_rows(container: str, database: str, identifiers: set[str]) -> dict[str, Any]:
|
|
empty = {table: [] for table in CANONICAL_TABLES}
|
|
if not identifiers:
|
|
return empty
|
|
values = ", ".join(_sql_text(value) for value in sorted(identifiers))
|
|
pairs = []
|
|
for table in CANONICAL_TABLES:
|
|
pairs.append(
|
|
f"""'{table}', coalesce((
|
|
select jsonb_agg(to_jsonb(row_data) order by to_jsonb(row_data)::text)
|
|
from {table} row_data
|
|
where exists (
|
|
select 1
|
|
from jsonb_each_text(to_jsonb(row_data)) item
|
|
where item.value = any(array[{values}]::text[])
|
|
)
|
|
), '[]'::jsonb)"""
|
|
)
|
|
return _psql_json(
|
|
container,
|
|
database,
|
|
"""\\set ON_ERROR_STOP on
|
|
begin transaction read only;
|
|
select jsonb_build_object(
|
|
"""
|
|
+ ",\n ".join(pairs)
|
|
+ """
|
|
)::text;
|
|
rollback;
|
|
""",
|
|
)
|
|
|
|
|
|
class ProfileCopyPolicy:
|
|
"""Exclude runtime state and credentials without reading their contents."""
|
|
|
|
OMITTED_DIRECTORIES = frozenset(
|
|
{
|
|
".claude",
|
|
".codex",
|
|
".git",
|
|
".pytest_cache",
|
|
"__pycache__",
|
|
"auth",
|
|
"backups",
|
|
"cache",
|
|
"credentials",
|
|
"image_cache",
|
|
"logs",
|
|
"memory_backups",
|
|
"secrets",
|
|
"sessions",
|
|
"state",
|
|
}
|
|
)
|
|
OMITTED_EXACT_FILES = frozenset(
|
|
{
|
|
".env",
|
|
"auth.json",
|
|
"auth.lock",
|
|
"credentials.json",
|
|
"gateway.pid",
|
|
}
|
|
)
|
|
STATE_SUFFIXES = (".db", ".db-shm", ".db-wal", ".sqlite", ".sqlite3", ".sqlite-shm", ".sqlite-wal")
|
|
SECRET_SUFFIXES = (".key", ".pem", ".p12", ".pfx")
|
|
SENSITIVE_NAME_MARKERS = ("auth", "credential", "secret", "token")
|
|
SENSITIVE_METADATA_SUFFIXES = ("", ".json", ".jsonl", ".toml", ".yaml", ".yml", ".txt")
|
|
|
|
def __init__(self) -> None:
|
|
self.omitted_counts: dict[str, int] = {}
|
|
|
|
def _omit(self, category: str) -> bool:
|
|
self.omitted_counts[category] = self.omitted_counts.get(category, 0) + 1
|
|
return True
|
|
|
|
def category(self, path: Path) -> str | None:
|
|
name = path.name.lower()
|
|
if path.is_symlink():
|
|
return "symlink"
|
|
if path.is_dir() and name in self.OMITTED_DIRECTORIES:
|
|
return "runtime_or_sensitive_directory"
|
|
if name in self.OMITTED_EXACT_FILES or name.startswith(".env.") or name.endswith(".env"):
|
|
return "environment_or_auth_file"
|
|
if any(marker in name for marker in self.SENSITIVE_NAME_MARKERS) and path.suffix.lower() in (
|
|
self.SENSITIVE_METADATA_SUFFIXES
|
|
):
|
|
return "credential_file"
|
|
if name.startswith("state."):
|
|
return "state_database"
|
|
if "backup" in name or ".bak-" in name or name.endswith((".bak", ".old")):
|
|
return "backup"
|
|
if name.endswith(self.STATE_SUFFIXES):
|
|
return "state_database"
|
|
if name.endswith(self.SECRET_SUFFIXES):
|
|
return "credential_file"
|
|
if name.endswith(".lock"):
|
|
return "lock"
|
|
return None
|
|
|
|
def __call__(self, directory: str, names: list[str]) -> set[str]:
|
|
ignored: set[str] = set()
|
|
parent = Path(directory)
|
|
for name in names:
|
|
category = self.category(parent / name)
|
|
if category and self._omit(category):
|
|
ignored.add(name)
|
|
return ignored
|
|
|
|
|
|
def _safe_name(value: str, fallback: str) -> str:
|
|
normalized = re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-.")
|
|
return (normalized or fallback)[:48]
|
|
|
|
|
|
def register_temp_root(path: Path) -> None:
|
|
_ACTIVE_TEMP_ROOTS.add(path)
|
|
|
|
|
|
def unregister_temp_root(path: Path | None) -> None:
|
|
if path is not None:
|
|
_ACTIVE_TEMP_ROOTS.discard(path)
|
|
|
|
|
|
def temp_path_absent(path: Path | None) -> bool:
|
|
return path is None or not os.path.lexists(path)
|
|
|
|
|
|
def copy_profile(
|
|
*,
|
|
run_id: str,
|
|
prompt_id: str,
|
|
live_profile: Path | None = None,
|
|
) -> tuple[Path, dict[str, Any]]:
|
|
live_profile = live_profile or LIVE_PROFILE
|
|
if not live_profile.is_dir():
|
|
raise CheckpointError(f"live leoclean profile does not exist: {live_profile}")
|
|
unique = uuid.uuid4().hex[:12]
|
|
prefix = f"leo-clone-checkpoint-{_safe_name(run_id, 'run')}-{_safe_name(prompt_id, 'prompt')}-{unique}-"
|
|
temp_root = Path(tempfile.mkdtemp(prefix=prefix))
|
|
register_temp_root(temp_root)
|
|
temp_root.chmod(0o700)
|
|
target = temp_root / "profile"
|
|
policy = ProfileCopyPolicy()
|
|
try:
|
|
shutil.copytree(live_profile, target, symlinks=False, ignore=policy)
|
|
audit = audit_temp_profile(target, policy)
|
|
if not audit["passes"]:
|
|
raise CheckpointError("temporary profile retained a forbidden state or credential artifact")
|
|
except BaseException:
|
|
remove_tree(temp_root)
|
|
raise
|
|
return target, audit
|
|
|
|
|
|
def copy_ephemeral_model_auth(
|
|
temp_profile: Path,
|
|
*,
|
|
live_profile: Path | None = None,
|
|
) -> dict[str, Any]:
|
|
live_profile = live_profile or LIVE_PROFILE
|
|
source = live_profile / "auth.json"
|
|
target = temp_profile / "auth.json"
|
|
if not source.is_file() or source.is_symlink():
|
|
raise CheckpointError(f"live model auth file is unavailable or unsafe: {source}")
|
|
if target.exists() or target.is_symlink():
|
|
raise CheckpointError(f"temporary model auth target already exists: {target}")
|
|
shutil.copyfile(source, target)
|
|
target.chmod(0o600)
|
|
return {
|
|
"mode": "ephemeral_file_copy",
|
|
"source_path": str(source),
|
|
"temporary_path": str(target),
|
|
"temporary_mode": oct(target.stat().st_mode & 0o777),
|
|
"source_contents_recorded": False,
|
|
"source_fingerprint_recorded": False,
|
|
}
|
|
|
|
|
|
def load_ephemeral_provider_environment(temp_profile: Path) -> tuple[dict[str, str], dict[str, Any]]:
|
|
"""Resolve only env-backed credentials for the profile's configured provider."""
|
|
try:
|
|
import yaml
|
|
|
|
config = yaml.safe_load((temp_profile / "config.yaml").read_text(encoding="utf-8")) or {}
|
|
auth = json.loads((temp_profile / "auth.json").read_text(encoding="utf-8"))
|
|
except (OSError, ValueError, TypeError) as exc:
|
|
raise CheckpointError(f"cannot resolve temporary provider credential binding: {exc}") from exc
|
|
|
|
model_config = config.get("model") if isinstance(config, dict) else None
|
|
configured_provider = model_config.get("provider") if isinstance(model_config, dict) else None
|
|
provider = str(configured_provider or auth.get("active_provider") or "").strip().lower()
|
|
pool = auth.get("credential_pool") if isinstance(auth, dict) else None
|
|
entries = pool.get(provider, []) if isinstance(pool, dict) else []
|
|
if not isinstance(entries, list):
|
|
raise CheckpointError(f"temporary credential pool for provider {provider!r} is not a list")
|
|
|
|
environment: dict[str, str] = {}
|
|
sources: list[str] = []
|
|
ordered_entries = sorted(
|
|
(entry for entry in entries if isinstance(entry, dict)),
|
|
key=lambda entry: int(entry.get("priority", 0) or 0),
|
|
)
|
|
for entry in ordered_entries:
|
|
source = str(entry.get("source") or "")
|
|
if not source.startswith("env:"):
|
|
continue
|
|
name = source.removeprefix("env:").strip()
|
|
if not SAFE_ENV_NAME_RE.fullmatch(name):
|
|
raise CheckpointError(f"unsafe provider credential environment name: {name!r}")
|
|
if not (name.endswith("_API_KEY") or name.endswith("_TOKEN")):
|
|
raise CheckpointError(f"provider credential environment name is outside the allowlist: {name!r}")
|
|
value = str(entry.get("access_token") or entry.get("api_key") or "")
|
|
if value and name not in environment:
|
|
environment[name] = value
|
|
sources.append(source)
|
|
|
|
return environment, {
|
|
"configured_provider": provider or None,
|
|
"environment_names": sorted(environment),
|
|
"bound_credential_count": len(environment),
|
|
"sources": sorted(sources),
|
|
"credential_contents_recorded": False,
|
|
"credential_fingerprints_recorded": False,
|
|
}
|
|
|
|
|
|
def audit_temp_profile(profile: Path, policy: ProfileCopyPolicy | None = None) -> dict[str, Any]:
|
|
policy = policy or ProfileCopyPolicy()
|
|
forbidden_categories: dict[str, int] = {}
|
|
file_count = 0
|
|
byte_count = 0
|
|
for path in profile.rglob("*"):
|
|
category = policy.category(path)
|
|
if category:
|
|
forbidden_categories[category] = forbidden_categories.get(category, 0) + 1
|
|
if path.is_file() and not path.is_symlink():
|
|
file_count += 1
|
|
byte_count += path.stat().st_size
|
|
return {
|
|
"passes": not forbidden_categories,
|
|
"copied_file_count": file_count,
|
|
"copied_bytes": byte_count,
|
|
"omitted_counts_by_category": dict(sorted(policy.omitted_counts.items())),
|
|
"forbidden_artifact_counts_by_category": dict(sorted(forbidden_categories.items())),
|
|
"secret_contents_read_or_recorded": False,
|
|
}
|
|
|
|
|
|
def _detach_and_write(path: Path, content: str, *, mode: int) -> None:
|
|
if path.is_symlink():
|
|
path.unlink()
|
|
elif path.exists() and not path.is_file():
|
|
raise CheckpointError(f"temporary bridge path is not a regular file: {path}")
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(content, encoding="utf-8")
|
|
path.chmod(mode)
|
|
|
|
|
|
def build_forced_bridge_wrapper(
|
|
*,
|
|
kb_tool: Path,
|
|
tool_log: Path,
|
|
container: str,
|
|
database: str,
|
|
run_nonce: str,
|
|
) -> str:
|
|
python = str(Path(sys.executable).resolve())
|
|
return f"""#!/bin/bash
|
|
set -uo pipefail
|
|
|
|
KB_TOOL={shlex.quote(str(kb_tool))}
|
|
TOOL_LOG={shlex.quote(str(tool_log))}
|
|
TARGET_CONTAINER={shlex.quote(container)}
|
|
TARGET_DATABASE={shlex.quote(database)}
|
|
RUN_NONCE={shlex.quote(run_nonce)}
|
|
PYTHON={shlex.quote(python)}
|
|
DOCKER={shlex.quote(SYSTEM_DOCKER)}
|
|
export PATH={shlex.quote(SYSTEM_EXEC_PATH)}
|
|
INVOCATION_ID="$($PYTHON -c 'import uuid; print(uuid.uuid4())')"
|
|
DB_RECEIPT="$($DOCKER exec "$TARGET_CONTAINER" psql -U postgres -d "$TARGET_DATABASE" -At -q -v ON_ERROR_STOP=1 -c "select jsonb_build_object('current_database', current_database(), 'system_identifier', (select system_identifier::text from pg_control_system()), 'transaction_read_only', current_setting('transaction_read_only'))::text")"
|
|
if [[ -z "$DB_RECEIPT" ]]; then
|
|
exit 125
|
|
fi
|
|
|
|
$PYTHON - "$TOOL_LOG" "$RUN_NONCE" "$INVOCATION_ID" "$TARGET_CONTAINER" "$TARGET_DATABASE" "$DB_RECEIPT" "$@" <<'PY'
|
|
import json
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
path, run_nonce, invocation_id, container, database, db_receipt, *argv = sys.argv[1:]
|
|
event = {{
|
|
"phase": "start",
|
|
"at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"invocation_id": invocation_id,
|
|
"run_nonce": run_nonce,
|
|
"container": container,
|
|
"database": database,
|
|
"database_identity": json.loads(db_receipt),
|
|
"argv": argv,
|
|
}}
|
|
with open(path, "a", encoding="utf-8") as handle:
|
|
handle.write(json.dumps(event, sort_keys=True) + "\\n")
|
|
PY
|
|
|
|
$PYTHON "$KB_TOOL" --local --container "$TARGET_CONTAINER" --db "$TARGET_DATABASE" "$@"
|
|
STATUS=$?
|
|
|
|
$PYTHON - "$TOOL_LOG" "$RUN_NONCE" "$INVOCATION_ID" "$TARGET_CONTAINER" "$TARGET_DATABASE" "$STATUS" <<'PY'
|
|
import json
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
path, run_nonce, invocation_id, container, database, status = sys.argv[1:]
|
|
event = {{
|
|
"phase": "end",
|
|
"at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"invocation_id": invocation_id,
|
|
"run_nonce": run_nonce,
|
|
"container": container,
|
|
"database": database,
|
|
"returncode": int(status),
|
|
}}
|
|
with open(path, "a", encoding="utf-8") as handle:
|
|
handle.write(json.dumps(event, sort_keys=True) + "\\n")
|
|
PY
|
|
|
|
exit "$STATUS"
|
|
"""
|
|
|
|
|
|
def inject_skill_binding(skill_text: str, binding: str) -> str:
|
|
if not skill_text.startswith("---\n"):
|
|
return binding.rstrip() + "\n\n" + skill_text
|
|
frontmatter_end = skill_text.find("\n---\n", 4)
|
|
if frontmatter_end < 0:
|
|
raise CheckpointError("temporary teleo-kb bridge skill has unterminated YAML frontmatter")
|
|
insertion = frontmatter_end + len("\n---\n")
|
|
return skill_text[:insertion] + "\n" + binding.rstrip() + "\n\n" + skill_text[insertion:]
|
|
|
|
|
|
def patch_temp_bridge(temp_profile: Path, container: str, database: str) -> dict[str, Any]:
|
|
wrapper = temp_profile / "bin" / "teleo-kb"
|
|
kb_tool = temp_profile / "bin" / "kb_tool.py"
|
|
bridge_skill = temp_profile / "skills" / "teleo-kb-bridge" / "SKILL.md"
|
|
tool_log = temp_profile / "checkpoint-tool-calls.jsonl"
|
|
run_nonce = uuid.uuid4().hex
|
|
|
|
if not wrapper.exists() and not wrapper.is_symlink():
|
|
raise CheckpointError(f"temporary teleo-kb wrapper is missing: {wrapper}")
|
|
if not kb_tool.exists():
|
|
raise CheckpointError(f"temporary kb_tool.py is missing: {kb_tool}")
|
|
if not bridge_skill.exists():
|
|
raise CheckpointError(f"temporary teleo-kb bridge skill is missing: {bridge_skill}")
|
|
|
|
skill_text = bridge_skill.read_text(encoding="utf-8")
|
|
temporary_path = str(wrapper)
|
|
for live_wrapper in {
|
|
"/home/teleo/.hermes/profiles/leoclean/bin/teleo-kb",
|
|
str(LIVE_PROFILE / "bin" / "teleo-kb"),
|
|
}:
|
|
skill_text = skill_text.replace(live_wrapper, "teleo-kb")
|
|
binding = f"""
|
|
## Disposable Checkpoint Binding
|
|
|
|
This temporary profile is bound only to `{container}` / `{database}`. For all
|
|
KB reads in this session, start a terminal call directly with `teleo-kb`; the
|
|
checkpoint PATH resolves that name only to `{temporary_path}`. Run exactly one
|
|
KB command per terminal call. Do not define shell variables or use `exec`,
|
|
pipes, redirects, shell operators, `docker`, `psql`, cloud database tools, or
|
|
wrappers under the live profile. This checkpoint exposes only read-only bridge
|
|
verbs and records every invocation in its receipt.
|
|
|
|
End the final answer with one machine-readable line using the full proposal
|
|
UUID and the row state you observed:
|
|
`KB_STATE: {{"proposal_id":"<uuid>","status":"pending_review|approved|applied","applied":false}}`
|
|
|
|
"""
|
|
skill_text = inject_skill_binding(skill_text, binding.lstrip())
|
|
wrapper_text = build_forced_bridge_wrapper(
|
|
kb_tool=kb_tool,
|
|
tool_log=tool_log,
|
|
container=container,
|
|
database=database,
|
|
run_nonce=run_nonce,
|
|
)
|
|
|
|
try:
|
|
descriptor = os.open(
|
|
tool_log,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
|
0o600,
|
|
)
|
|
except FileExistsError as exc:
|
|
raise CheckpointError("temporary tool log must not pre-exist") from exc
|
|
os.close(descriptor)
|
|
_detach_and_write(wrapper, wrapper_text, mode=0o700)
|
|
_detach_and_write(bridge_skill, skill_text, mode=0o600)
|
|
return {
|
|
"wrapper_path": str(wrapper),
|
|
"bridge_skill_path": str(bridge_skill),
|
|
"tool_log_path": str(tool_log),
|
|
"wrapper_sha256": sha256_bytes(wrapper_text.encode()),
|
|
"bridge_skill_sha256": sha256_bytes(skill_text.encode()),
|
|
"forced_container": container,
|
|
"forced_database": database,
|
|
"run_nonce": run_nonce,
|
|
"patched_files": [str(wrapper), str(bridge_skill)],
|
|
}
|
|
|
|
|
|
def bridge_hashes(profile: Path | None = None) -> dict[str, str | None]:
|
|
profile = profile or LIVE_PROFILE
|
|
paths = {
|
|
"wrapper": profile / "bin" / "teleo-kb",
|
|
"bridge_skill": profile / "skills" / "teleo-kb-bridge" / "SKILL.md",
|
|
}
|
|
result: dict[str, str | None] = {}
|
|
for name, path in paths.items():
|
|
try:
|
|
result[name] = sha256_bytes(path.read_bytes())
|
|
except OSError:
|
|
result[name] = None
|
|
return result
|
|
|
|
|
|
@contextmanager
|
|
def gateway_environment(
|
|
temp_profile: Path,
|
|
*,
|
|
chat_id: str,
|
|
user_id: str,
|
|
provider_environment: dict[str, str] | None = None,
|
|
) -> Iterator[None]:
|
|
updates = {
|
|
"HERMES_HOME": str(temp_profile),
|
|
"HOME": str(LIVE_PROFILE.parents[2]),
|
|
"PATH": f"{temp_profile / 'bin'}:{os.environ.get('PATH', '')}",
|
|
"TELEO_KB_MODE": "local",
|
|
"TELEGRAM_ALLOWED_CHATS": chat_id,
|
|
"TELEGRAM_ALLOWED_USERS": user_id,
|
|
"TELEGRAM_ALLOW_ALL_USERS": "false",
|
|
"GATEWAY_ALLOW_ALL_USERS": "false",
|
|
}
|
|
provider_environment = provider_environment or {}
|
|
collisions = sorted(set(updates) & set(provider_environment))
|
|
if collisions:
|
|
raise CheckpointError(f"provider credential environment collides with safety settings: {collisions}")
|
|
updates.update(provider_environment)
|
|
old = {key: os.environ.get(key) for key in updates}
|
|
original_sys_path = list(sys.path)
|
|
try:
|
|
os.environ.update(updates)
|
|
sys.path.insert(0, str(AGENT_ROOT))
|
|
yield
|
|
finally:
|
|
sys.path[:] = original_sys_path
|
|
for key, value in old.items():
|
|
if value is None:
|
|
os.environ.pop(key, None)
|
|
else:
|
|
os.environ[key] = value
|
|
|
|
|
|
async def _close_runner(runner: Any) -> dict[str, Any]:
|
|
for method_name in ("aclose", "close"):
|
|
method = getattr(runner, method_name, None)
|
|
if not callable(method):
|
|
continue
|
|
try:
|
|
result = method()
|
|
if inspect.isawaitable(result):
|
|
await asyncio.wait_for(result, timeout=15)
|
|
return {"attempted": True, "method": method_name, "ok": True}
|
|
except Exception as exc: # pragma: no cover - depends on live runner implementation
|
|
return {"attempted": True, "method": method_name, "ok": False, "error": str(exc)}
|
|
return {"attempted": False, "method": None, "ok": True}
|
|
|
|
|
|
def transcript_tool_trace(
|
|
messages: list[dict[str, Any]],
|
|
*,
|
|
content_limit: int = 4000,
|
|
argument_limit: int = 2048,
|
|
) -> list[dict[str, Any]]:
|
|
"""Retain bounded tool-call evidence without copying the full conversation."""
|
|
trace: list[dict[str, Any]] = []
|
|
for message in messages:
|
|
role = message.get("role")
|
|
tool_calls = message.get("tool_calls")
|
|
if role == "assistant" and isinstance(tool_calls, list):
|
|
for call in tool_calls:
|
|
if not isinstance(call, dict):
|
|
continue
|
|
function = call.get("function") if isinstance(call.get("function"), dict) else {}
|
|
raw_arguments = str(function.get("arguments") or call.get("arguments") or "")
|
|
trace.append(
|
|
{
|
|
"phase": "call",
|
|
"tool_call_id": call.get("id"),
|
|
"tool_name": function.get("name") or call.get("name"),
|
|
"arguments": redact_text(raw_arguments)[:argument_limit],
|
|
"arguments_truncated": len(raw_arguments) > argument_limit,
|
|
}
|
|
)
|
|
elif role == "tool":
|
|
raw_content = str(message.get("content") or "")
|
|
content = redact_text(raw_content)
|
|
trace.append(
|
|
{
|
|
"phase": "result",
|
|
"tool_call_id": message.get("tool_call_id"),
|
|
"tool_name": message.get("tool_name") or message.get("name"),
|
|
"content": content[:content_limit],
|
|
"content_truncated": len(raw_content) > content_limit or len(content) > content_limit,
|
|
}
|
|
)
|
|
return redact_value(trace)
|
|
|
|
|
|
def transcript_terminal_attempt_summary(events: list[dict[str, Any]]) -> dict[str, Any]:
|
|
terminal_calls = {
|
|
str(event.get("tool_call_id")): event
|
|
for event in events
|
|
if event.get("phase") == "call" and event.get("tool_name") == "terminal"
|
|
}
|
|
result_content = {
|
|
str(event.get("tool_call_id")): str(event.get("content") or "")
|
|
for event in events
|
|
if event.get("phase") == "result"
|
|
}
|
|
known_exit_codes: dict[str, int] = {}
|
|
for call_id in terminal_calls:
|
|
match = re.search(r'"exit_code"\s*:\s*(-?\d+)', result_content.get(call_id, ""))
|
|
if match:
|
|
known_exit_codes[call_id] = int(match.group(1))
|
|
rejected = sorted(call_id for call_id, exit_code in known_exit_codes.items() if exit_code == 126)
|
|
nonzero = sorted(call_id for call_id, exit_code in known_exit_codes.items() if exit_code != 0)
|
|
return {
|
|
"call_count": len(terminal_calls),
|
|
"known_exit_codes": known_exit_codes,
|
|
"rejected_call_ids": rejected,
|
|
"rejected_call_count": len(rejected),
|
|
"nonzero_call_ids": nonzero,
|
|
"nonzero_call_count": len(nonzero),
|
|
"result_code_unknown_call_count": len(terminal_calls) - len(known_exit_codes),
|
|
}
|
|
|
|
|
|
class BridgeArgumentParser(argparse.ArgumentParser):
|
|
def error(self, message: str) -> None:
|
|
raise CheckpointError(message)
|
|
|
|
def exit(self, status: int = 0, message: str | None = None) -> None:
|
|
raise CheckpointError(message or f"argument parser exited with status {status}")
|
|
|
|
|
|
def _uuid_argument(value: str) -> str:
|
|
try:
|
|
return str(uuid.UUID(value))
|
|
except ValueError as exc:
|
|
raise argparse.ArgumentTypeError("expected a UUID") from exc
|
|
|
|
|
|
def validate_read_only_bridge_argv(argv: list[str]) -> None:
|
|
parser = BridgeArgumentParser(add_help=False)
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
def command(name: str) -> argparse.ArgumentParser:
|
|
item = subparsers.add_parser(name, add_help=False)
|
|
item.add_argument("--format", choices=("markdown", "json"), default="markdown")
|
|
return item
|
|
|
|
for name in ("search", "context"):
|
|
item = command(name)
|
|
item.add_argument("query")
|
|
item.add_argument("--limit", type=int, default=5)
|
|
if name == "search":
|
|
item.add_argument("--context-limit", type=int, default=5)
|
|
else:
|
|
item.add_argument("--context-limit", type=int, default=6)
|
|
show = command("show")
|
|
show.add_argument("claim_id", type=_uuid_argument)
|
|
for name in ("evidence", "edges"):
|
|
item = command(name)
|
|
item.add_argument("claim_id", type=_uuid_argument)
|
|
item.add_argument("--limit", type=int, default=12)
|
|
list_proposals = command("list-proposals")
|
|
list_proposals.add_argument(
|
|
"--status",
|
|
choices=("pending_review", "approved", "applied", "canceled", "all"),
|
|
default="pending_review",
|
|
)
|
|
list_proposals.add_argument("--limit", type=int, default=10)
|
|
search_proposals = command("search-proposals")
|
|
search_proposals.add_argument("query")
|
|
search_proposals.add_argument(
|
|
"--status",
|
|
choices=("pending_review", "approved", "applied", "canceled", "all"),
|
|
default="all",
|
|
)
|
|
search_proposals.add_argument("--limit", type=int, default=10)
|
|
show_proposal = command("show-proposal")
|
|
show_proposal.add_argument("proposal_id", type=_uuid_argument)
|
|
command("decision-matrix-status")
|
|
|
|
parsed = parser.parse_args(argv)
|
|
for name, value in vars(parsed).items():
|
|
if name.endswith("limit") and not 1 <= value <= 100:
|
|
raise CheckpointError(f"{name.replace('_', '-')} must be between 1 and 100")
|
|
|
|
|
|
def runner_tool_trace(runner: Any, session_key: str) -> dict[str, Any]:
|
|
entry = getattr(getattr(runner, "session_store", None), "_entries", {}).get(session_key)
|
|
session_id = getattr(entry, "session_id", None)
|
|
if not session_id:
|
|
return {"session_id": None, "events": [], "event_count": 0}
|
|
messages = runner.session_store.load_transcript(session_id)
|
|
events = transcript_tool_trace(messages)
|
|
return {"session_id": session_id, "events": events, "event_count": len(events)}
|
|
|
|
|
|
def restricted_bridge_terminal_handler(
|
|
wrapper: Path,
|
|
temp_profile: Path,
|
|
tool_args: dict[str, Any],
|
|
**_kwargs: Any,
|
|
) -> str:
|
|
command = str(tool_args.get("command") or "")
|
|
if len(command) > 8192:
|
|
return json.dumps({"output": "", "exit_code": 126, "error": "checkpoint command is too long"})
|
|
if tool_args.get("background") or tool_args.get("pty") or tool_args.get("workdir"):
|
|
return json.dumps(
|
|
{"output": "", "exit_code": 126, "error": "background, PTY, and workdir overrides are disabled"}
|
|
)
|
|
try:
|
|
argv = shlex.split(command)
|
|
except ValueError as exc:
|
|
return json.dumps({"output": "", "exit_code": 126, "error": f"invalid command quoting: {exc}"})
|
|
while argv and argv[-1] in HARMLESS_TRAILING_REDIRECT_TOKENS:
|
|
argv.pop()
|
|
command_path = f"{temp_profile / 'bin'}:{os.environ.get('PATH', '')}"
|
|
try:
|
|
exact_wrapper = wrapper.resolve(strict=True)
|
|
executable = shutil.which(argv[0], path=command_path) if argv else None
|
|
invoked = Path(executable).resolve(strict=True) if executable else None
|
|
except OSError:
|
|
invoked = None
|
|
exact_wrapper = wrapper.resolve(strict=False)
|
|
if invoked != exact_wrapper:
|
|
return json.dumps(
|
|
{"output": "", "exit_code": 126, "error": "only the clone-bound teleo-kb wrapper is available"}
|
|
)
|
|
if len(argv) < 2 or argv[1] not in READ_ONLY_BRIDGE_COMMANDS:
|
|
return json.dumps(
|
|
{"output": "", "exit_code": 126, "error": "only read-only Teleo KB bridge commands are available"}
|
|
)
|
|
try:
|
|
validate_read_only_bridge_argv(argv[1:])
|
|
except CheckpointError as exc:
|
|
return json.dumps({"output": "", "exit_code": 126, "error": f"invalid read-only KB arguments: {exc}"})
|
|
timeout = min(max(int(tool_args.get("timeout") or 60), 1), 120)
|
|
child_environment = {
|
|
"HOME": str(temp_profile),
|
|
"PATH": command_path,
|
|
"TELEO_KB_MODE": "local",
|
|
}
|
|
try:
|
|
proc = subprocess.run(
|
|
argv,
|
|
cwd=temp_profile,
|
|
env=child_environment,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
timeout=timeout,
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
return json.dumps({"output": "", "exit_code": 124, "error": f"bridge command exceeded {timeout}s"})
|
|
return json.dumps(
|
|
{
|
|
"output": redact_text(proc.stdout),
|
|
"exit_code": proc.returncode,
|
|
"error": redact_text(proc.stderr) or None,
|
|
}
|
|
)
|
|
|
|
|
|
def install_checkpoint_tool_surface(temp_profile: Path) -> dict[str, Any]:
|
|
import tools.skills_tool
|
|
import tools.terminal_tool # noqa: F401
|
|
import toolsets
|
|
from hermes_cli import tools_config
|
|
from tools.registry import registry
|
|
|
|
toolset_name = "checkpoint-kb-readonly"
|
|
allowed_tools = ["skills_list", "skill_view", "terminal"]
|
|
toolsets.TOOLSETS[toolset_name] = {
|
|
"description": "Read-only clone-bound KB checkpoint tools",
|
|
"tools": allowed_tools,
|
|
"includes": [],
|
|
}
|
|
tools_config._get_platform_tools = lambda *_args, **_kwargs: {toolset_name}
|
|
missing_tools = sorted(name for name in allowed_tools if name not in registry._tools)
|
|
if missing_tools:
|
|
raise CheckpointError(f"required checkpoint tools are not registered: {missing_tools}")
|
|
allowed_entries = {name: registry._tools[name] for name in allowed_tools}
|
|
registry._tools.clear()
|
|
registry._tools.update(allowed_entries)
|
|
terminal_entry = registry._tools.get("terminal")
|
|
if terminal_entry is None:
|
|
raise CheckpointError("Hermes terminal tool is not registered")
|
|
wrapper = temp_profile / "bin" / "teleo-kb"
|
|
terminal_entry.handler = lambda tool_args, **kwargs: restricted_bridge_terminal_handler(
|
|
wrapper,
|
|
temp_profile,
|
|
tool_args,
|
|
**kwargs,
|
|
)
|
|
return {
|
|
"toolsets": [toolset_name],
|
|
"allowed_tools": allowed_tools,
|
|
"actual_registry_tools": sorted(registry._tools),
|
|
"actual_registry_tool_types": {
|
|
name: f"{entry.__class__.__module__}.{entry.__class__.__name__}" for name, entry in registry._tools.items()
|
|
},
|
|
"actual_registry_tool_fields": {
|
|
name: sorted(vars(entry)) if hasattr(entry, "__dict__") else [] for name, entry in registry._tools.items()
|
|
},
|
|
"actual_registry_tool_schema_sha256": {
|
|
name: json_sha256(
|
|
{
|
|
field: redact_text(repr(getattr(entry, field)))[:4096]
|
|
for field in ("name", "description", "parameters", "schema", "input_schema")
|
|
if hasattr(entry, field)
|
|
}
|
|
)
|
|
for name, entry in registry._tools.items()
|
|
},
|
|
"read_only_bridge_commands": sorted(READ_ONLY_BRIDGE_COMMANDS),
|
|
"send_message_tool_enabled": "send_message" in registry._tools,
|
|
"terminal_restricted_to_clone_wrapper": True,
|
|
"terminal_subprocess_inherits_provider_credentials": False,
|
|
}
|
|
|
|
|
|
async def invoke_gateway(
|
|
args: argparse.Namespace,
|
|
temp_profile: Path,
|
|
*,
|
|
provider_environment: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
with gateway_environment(
|
|
temp_profile,
|
|
chat_id=args.chat_id,
|
|
user_id=args.user_id,
|
|
provider_environment=provider_environment,
|
|
):
|
|
from gateway.config import Platform
|
|
from gateway.platforms.base import MessageEvent, MessageType
|
|
from gateway.run import GatewayRunner, _resolve_gateway_model, _resolve_runtime_agent_kwargs
|
|
from gateway.session import SessionSource
|
|
|
|
tool_surface = install_checkpoint_tool_surface(temp_profile)
|
|
|
|
source = SessionSource(
|
|
platform=Platform.TELEGRAM,
|
|
chat_id=args.chat_id,
|
|
chat_name="Leo",
|
|
chat_type="group",
|
|
user_id=args.user_id,
|
|
user_name=args.user_name,
|
|
)
|
|
runner = GatewayRunner()
|
|
adapters = getattr(runner, "adapters", None)
|
|
if not isinstance(adapters, dict):
|
|
raise CheckpointError("GatewayRunner adapters could not be verified as a mapping")
|
|
tool_surface["gateway_adapters_verified_mapping"] = True
|
|
tool_surface["gateway_adapter_names"] = sorted(str(name) for name in adapters)
|
|
tool_surface["gateway_adapter_count"] = len(adapters)
|
|
gateway_model = _resolve_gateway_model()
|
|
runtime = _resolve_runtime_agent_kwargs()
|
|
turn_route = runner._resolve_turn_agent_config(args.prompt, gateway_model, runtime)
|
|
turn_runtime = turn_route.get("runtime") or {}
|
|
session_key = runner._session_key_for_source(source)
|
|
authorized = bool(runner._is_user_authorized(source))
|
|
result: dict[str, Any] = {
|
|
"runner_class": f"{runner.__class__.__module__}.{runner.__class__.__name__}",
|
|
"session_key": session_key,
|
|
"authorized": authorized,
|
|
"authorization_mode": "harness_exact_chat_and_user",
|
|
"handler_invoked": False,
|
|
"posted_to_telegram": False,
|
|
"model_free_fallback_used": False,
|
|
"tool_surface": tool_surface,
|
|
"routing": {
|
|
"gateway_model": gateway_model,
|
|
"turn_model": turn_route.get("model"),
|
|
"provider": turn_runtime.get("provider"),
|
|
"api_mode": turn_runtime.get("api_mode"),
|
|
"base_url_host": urlsplit(str(turn_runtime.get("base_url") or "")).hostname,
|
|
"api_key_present": bool(turn_runtime.get("api_key")),
|
|
"credential_pool_present": turn_runtime.get("credential_pool") is not None,
|
|
},
|
|
}
|
|
try:
|
|
if not authorized:
|
|
raise CheckpointError("Telegram-shaped source is not authorized by the copied live profile")
|
|
event = MessageEvent(
|
|
text=args.prompt,
|
|
message_type=MessageType.TEXT,
|
|
source=source,
|
|
message_id=f"clone-checkpoint-{args.run_id}-{args.prompt_id}",
|
|
)
|
|
result["started_at_utc"] = utc_now()
|
|
result["handler_invoked"] = True
|
|
reply = await runner._handle_message(event)
|
|
result["ended_at_utc"] = utc_now()
|
|
result["reply"] = reply if isinstance(reply, str) else str(reply or "")
|
|
result["transcript_tool_trace"] = runner_tool_trace(runner, session_key)
|
|
result["transcript_terminal_attempts"] = transcript_terminal_attempt_summary(
|
|
result["transcript_tool_trace"]["events"]
|
|
)
|
|
return result
|
|
finally:
|
|
result["runner_cleanup"] = await _close_runner(runner)
|
|
|
|
|
|
def process_group_alive(process: Any) -> bool:
|
|
pid = getattr(process, "pid", None)
|
|
if not pid:
|
|
return False
|
|
try:
|
|
os.killpg(pid, 0)
|
|
except ProcessLookupError:
|
|
return False
|
|
except PermissionError:
|
|
return True
|
|
return True
|
|
|
|
|
|
def _signal_gateway_process_group(process: Any, signum: signal.Signals) -> None:
|
|
try:
|
|
os.killpg(process.pid, signum)
|
|
except ProcessLookupError:
|
|
if process.is_alive():
|
|
if signum == signal.SIGKILL:
|
|
process.kill()
|
|
else:
|
|
process.terminate()
|
|
|
|
|
|
def _wait_for_process_group_exit(process: Any, timeout: float) -> bool:
|
|
deadline = time.monotonic() + timeout
|
|
while process_group_alive(process) and time.monotonic() < deadline:
|
|
time.sleep(0.02)
|
|
return not process_group_alive(process)
|
|
|
|
|
|
def terminate_gateway_child(process: Any, *, grace_seconds: float = 2.0) -> dict[str, Any]:
|
|
attempted = bool(process and (process.is_alive() or process_group_alive(process)))
|
|
terminated = False
|
|
killed = False
|
|
if attempted:
|
|
_signal_gateway_process_group(process, signal.SIGTERM)
|
|
process.join(timeout=grace_seconds)
|
|
group_exited = _wait_for_process_group_exit(process, grace_seconds)
|
|
terminated = not process.is_alive() and group_exited
|
|
if process and (process.is_alive() or process_group_alive(process)):
|
|
_signal_gateway_process_group(process, signal.SIGKILL)
|
|
process.join(timeout=grace_seconds)
|
|
_wait_for_process_group_exit(process, grace_seconds)
|
|
killed = True
|
|
return {
|
|
"attempted": attempted,
|
|
"terminated_after_sigterm": terminated,
|
|
"sigkill_used": killed,
|
|
"alive_after_cleanup": bool(process and process.is_alive()),
|
|
"process_group_alive_after_cleanup": bool(process and process_group_alive(process)),
|
|
"exitcode": getattr(process, "exitcode", None),
|
|
}
|
|
|
|
|
|
def cleanup_active_gateway_children() -> dict[int | None, dict[str, Any]]:
|
|
results: dict[int | None, dict[str, Any]] = {}
|
|
for process in tuple(_ACTIVE_GATEWAY_CHILDREN):
|
|
result = terminate_gateway_child(process)
|
|
results[getattr(process, "pid", None)] = result
|
|
if not result["alive_after_cleanup"] and not result["process_group_alive_after_cleanup"]:
|
|
_ACTIVE_GATEWAY_CHILDREN.discard(process)
|
|
return results
|
|
|
|
|
|
def _gateway_child_entry(
|
|
args: argparse.Namespace,
|
|
temp_profile: Path,
|
|
provider_environment: dict[str, str],
|
|
result_path: Path,
|
|
readiness_path: Path,
|
|
) -> None:
|
|
os.setsid()
|
|
for signum in (signal.SIGINT, signal.SIGTERM, getattr(signal, "SIGHUP", None)):
|
|
if signum is not None:
|
|
signal.signal(signum, signal.SIG_DFL)
|
|
write_report(
|
|
readiness_path,
|
|
{"pid": os.getpid(), "process_group_id": os.getpgrp(), "ready_at_utc": utc_now()},
|
|
)
|
|
if hasattr(signal, "pthread_sigmask"):
|
|
signal.pthread_sigmask(
|
|
signal.SIG_UNBLOCK,
|
|
{item for item in (signal.SIGINT, signal.SIGTERM, getattr(signal, "SIGHUP", None)) if item is not None},
|
|
)
|
|
retained_environment = {
|
|
key: value
|
|
for key, value in os.environ.items()
|
|
if key in {"PATH", "LANG", "LC_ALL", "LC_CTYPE", "TZ", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "TMPDIR"}
|
|
}
|
|
os.environ.clear()
|
|
os.environ.update(retained_environment)
|
|
os.environ["HOME"] = "/home/teleo"
|
|
try:
|
|
result = asyncio.run(
|
|
invoke_gateway(
|
|
args,
|
|
temp_profile,
|
|
provider_environment=provider_environment,
|
|
)
|
|
)
|
|
except BaseException as exc:
|
|
result = {
|
|
"authorized": False,
|
|
"handler_invoked": False,
|
|
"posted_to_telegram": False,
|
|
"model_free_fallback_used": False,
|
|
"handler_error": {"type": type(exc).__name__, "message": str(exc)},
|
|
}
|
|
write_report(result_path, result, secret_values=tuple(provider_environment.values()))
|
|
|
|
|
|
async def invoke_gateway_subprocess(
|
|
args: argparse.Namespace,
|
|
temp_profile: Path,
|
|
*,
|
|
provider_environment: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
if "fork" not in multiprocessing.get_all_start_methods():
|
|
raise CheckpointError("the live VPS gateway checkpoint requires multiprocessing fork support")
|
|
context = multiprocessing.get_context("fork")
|
|
result_path = temp_profile / ".checkpoint-gateway-result.json"
|
|
readiness_path = temp_profile / ".checkpoint-gateway-ready.json"
|
|
result_path.unlink(missing_ok=True)
|
|
readiness_path.unlink(missing_ok=True)
|
|
process = context.Process(
|
|
target=_gateway_child_entry,
|
|
args=(args, temp_profile, dict(provider_environment or {}), result_path, readiness_path),
|
|
name=f"leo-clone-checkpoint-{_safe_name(args.prompt_id, 'prompt')}",
|
|
)
|
|
watched_signals = {
|
|
item for item in (signal.SIGINT, signal.SIGTERM, getattr(signal, "SIGHUP", None)) if item is not None
|
|
}
|
|
previous_mask = None
|
|
if hasattr(signal, "pthread_sigmask"):
|
|
previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, watched_signals)
|
|
try:
|
|
process.start()
|
|
_ACTIVE_GATEWAY_CHILDREN.add(process)
|
|
finally:
|
|
if previous_mask is not None:
|
|
signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask)
|
|
|
|
readiness: dict[str, Any] | None = None
|
|
readiness_deadline = asyncio.get_running_loop().time() + 30
|
|
while process.is_alive() and asyncio.get_running_loop().time() < readiness_deadline:
|
|
if readiness_path.is_file():
|
|
try:
|
|
readiness = json.loads(readiness_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
readiness = None
|
|
if readiness:
|
|
break
|
|
await asyncio.sleep(0.05)
|
|
if readiness is None and readiness_path.is_file():
|
|
try:
|
|
readiness = json.loads(readiness_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
readiness = None
|
|
deadline = asyncio.get_running_loop().time() + args.turn_timeout
|
|
timed_out = False
|
|
termination = {
|
|
"attempted": False,
|
|
"terminated_after_sigterm": False,
|
|
"sigkill_used": False,
|
|
"alive_after_cleanup": False,
|
|
"process_group_alive_after_cleanup": False,
|
|
"exitcode": None,
|
|
}
|
|
try:
|
|
while readiness is not None and process.is_alive() and asyncio.get_running_loop().time() < deadline:
|
|
await asyncio.sleep(0.1)
|
|
if process.is_alive() and (readiness is None or asyncio.get_running_loop().time() >= deadline):
|
|
timed_out = True
|
|
termination = terminate_gateway_child(process)
|
|
else:
|
|
process.join(timeout=1)
|
|
|
|
result = None
|
|
if result_path.is_file():
|
|
try:
|
|
result = json.loads(result_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
result = None
|
|
if result is None:
|
|
result = {
|
|
"authorized": False,
|
|
"handler_invoked": False,
|
|
"posted_to_telegram": False,
|
|
"model_free_fallback_used": False,
|
|
"handler_error": {
|
|
"type": "TimeoutError" if timed_out else "GatewayChildError",
|
|
"message": (
|
|
f"gateway child exceeded {args.turn_timeout}s and was terminated"
|
|
if timed_out
|
|
else f"gateway child exited without a result (exitcode={process.exitcode})"
|
|
),
|
|
},
|
|
}
|
|
result["child_process"] = {
|
|
"pid": process.pid,
|
|
"timed_out": timed_out,
|
|
"exitcode": process.exitcode,
|
|
"termination": termination,
|
|
"alive_after_readback": process.is_alive(),
|
|
"process_group_alive_after_readback": process_group_alive(process),
|
|
"result_transport": "private_temp_file",
|
|
"readiness": readiness,
|
|
"readiness_verified": bool(
|
|
readiness and readiness.get("pid") == process.pid and readiness.get("process_group_id") == process.pid
|
|
),
|
|
}
|
|
return result
|
|
finally:
|
|
if process.is_alive() or process_group_alive(process):
|
|
terminate_gateway_child(process)
|
|
if not process.is_alive() and not process_group_alive(process):
|
|
_ACTIVE_GATEWAY_CHILDREN.discard(process)
|
|
|
|
|
|
def read_tool_proof(
|
|
path: Path | None,
|
|
container: str,
|
|
database: str,
|
|
*,
|
|
run_nonce: str,
|
|
database_identity: dict[str, Any],
|
|
require_database_read_only: bool = True,
|
|
) -> dict[str, Any]:
|
|
events: list[dict[str, Any]] = []
|
|
parse_errors: list[str] = []
|
|
if path and path.exists():
|
|
for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
event = json.loads(line)
|
|
except json.JSONDecodeError as exc:
|
|
parse_errors.append(f"line {number}: {exc}")
|
|
continue
|
|
events.append(event)
|
|
|
|
starts = [event for event in events if event.get("phase") == "start"]
|
|
end_events = [event for event in events if event.get("phase") == "end"]
|
|
start_ids = [event.get("invocation_id") for event in starts]
|
|
end_ids = [event.get("invocation_id") for event in end_events]
|
|
duplicate_ids = sorted(
|
|
str(item) for item in set(start_ids + end_ids) if start_ids.count(item) > 1 or end_ids.count(item) > 1
|
|
)
|
|
ends = {event.get("invocation_id"): event for event in end_events}
|
|
invocations = []
|
|
for event in events:
|
|
if event.get("phase") != "start":
|
|
continue
|
|
end = ends.get(event.get("invocation_id"), {})
|
|
invocations.append(
|
|
{
|
|
"invocation_id": event.get("invocation_id"),
|
|
"started_at_utc": event.get("at_utc"),
|
|
"ended_at_utc": end.get("at_utc"),
|
|
"container": event.get("container"),
|
|
"database": event.get("database"),
|
|
"run_nonce": event.get("run_nonce"),
|
|
"database_identity": event.get("database_identity"),
|
|
"argv": [redact_text(str(item))[:512] for item in event.get("argv", [])[:32]],
|
|
"returncode": end.get("returncode"),
|
|
"end_run_nonce": end.get("run_nonce"),
|
|
}
|
|
)
|
|
expected_system_identifier = str(database_identity.get("system_identifier") or "")
|
|
complete_pairing = (
|
|
not duplicate_ids and len(starts) == len(end_events) and set(start_ids) == set(end_ids) and not parse_errors
|
|
)
|
|
all_bound = bool(invocations) and all(
|
|
item["container"] == container
|
|
and item["database"] == database
|
|
and item["run_nonce"] == run_nonce
|
|
and item["end_run_nonce"] == run_nonce
|
|
and str((item["database_identity"] or {}).get("current_database")) == database
|
|
and str((item["database_identity"] or {}).get("system_identifier")) == expected_system_identifier
|
|
and (
|
|
not require_database_read_only
|
|
or str((item["database_identity"] or {}).get("transaction_read_only")) == "on"
|
|
)
|
|
for item in invocations
|
|
)
|
|
return {
|
|
"parse_errors": parse_errors,
|
|
"event_count": len(events),
|
|
"duplicate_invocation_ids": duplicate_ids,
|
|
"complete_start_end_pairing": complete_pairing,
|
|
"invocations": invocations,
|
|
"invocation_count": len(invocations),
|
|
"all_bound_to_supplied_target": complete_pairing and all_bound,
|
|
"database_read_only_required": require_database_read_only,
|
|
"all_completed_successfully": complete_pairing
|
|
and bool(invocations)
|
|
and all(item["returncode"] == 0 for item in invocations),
|
|
}
|
|
|
|
|
|
def remove_tree(path: Path | None) -> bool:
|
|
if path is None or temp_path_absent(path):
|
|
unregister_temp_root(path)
|
|
return True
|
|
try:
|
|
shutil.rmtree(path)
|
|
except Exception:
|
|
for root, directories, files in os.walk(path, topdown=False):
|
|
for name in files:
|
|
item = Path(root) / name
|
|
try:
|
|
item.chmod(0o600)
|
|
item.unlink()
|
|
except OSError:
|
|
pass
|
|
for name in directories:
|
|
item = Path(root) / name
|
|
try:
|
|
item.chmod(0o700)
|
|
item.rmdir()
|
|
except OSError:
|
|
pass
|
|
try:
|
|
path.chmod(0o700)
|
|
path.rmdir()
|
|
except OSError:
|
|
pass
|
|
absent = temp_path_absent(path)
|
|
if absent:
|
|
unregister_temp_root(path)
|
|
return absent
|
|
|
|
|
|
def cleanup_active_temp_roots() -> dict[str, bool]:
|
|
return {str(path): remove_tree(path) and temp_path_absent(path) for path in tuple(_ACTIVE_TEMP_ROOTS)}
|
|
|
|
|
|
def _termination_handler(signum: int, _frame: Any) -> None:
|
|
global _LAST_TERMINATION_SIGNAL
|
|
_LAST_TERMINATION_SIGNAL = signum
|
|
child_results = cleanup_active_gateway_children()
|
|
if all(
|
|
not result["alive_after_cleanup"] and not result["process_group_alive_after_cleanup"]
|
|
for result in child_results.values()
|
|
):
|
|
cleanup_active_temp_roots()
|
|
raise TerminationRequested(signum)
|
|
|
|
|
|
@contextmanager
|
|
def termination_cleanup_handlers() -> Iterator[None]:
|
|
watched = tuple(
|
|
item for item in (signal.SIGINT, signal.SIGTERM, getattr(signal, "SIGHUP", None)) if item is not None
|
|
)
|
|
previous: dict[signal.Signals, Any] = {}
|
|
try:
|
|
for signum in watched:
|
|
previous[signum] = signal.getsignal(signum)
|
|
signal.signal(signum, _termination_handler)
|
|
yield
|
|
finally:
|
|
child_results = cleanup_active_gateway_children()
|
|
if all(
|
|
not result["alive_after_cleanup"] and not result["process_group_alive_after_cleanup"]
|
|
for result in child_results.values()
|
|
):
|
|
cleanup_active_temp_roots()
|
|
for signum, handler in previous.items():
|
|
signal.signal(signum, handler)
|
|
|
|
|
|
def _uuid_values(value: Any) -> set[str]:
|
|
found: set[str] = set()
|
|
if isinstance(value, dict):
|
|
for key, item in value.items():
|
|
found.update(_uuid_values(key))
|
|
found.update(_uuid_values(item))
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
found.update(_uuid_values(item))
|
|
elif isinstance(value, str):
|
|
found.update(match.group(0).lower() for match in UUID_RE.finditer(value))
|
|
return found
|
|
|
|
|
|
def _state_semantics(proposal: dict[str, Any], expected_state: str) -> dict[str, bool]:
|
|
status_matches = proposal.get("status") == expected_state
|
|
has_reviewer = bool(proposal.get("reviewed_by_handle") or proposal.get("reviewed_by_agent_id"))
|
|
has_applier = bool(proposal.get("applied_by_handle") or proposal.get("applied_by_agent_id"))
|
|
if expected_state == "pending_review":
|
|
review_matches = proposal.get("reviewed_at") is None and not has_reviewer
|
|
apply_matches = proposal.get("applied_at") is None and not has_applier
|
|
elif expected_state == "approved":
|
|
review_matches = bool(proposal.get("reviewed_at")) and has_reviewer
|
|
apply_matches = proposal.get("applied_at") is None and not has_applier
|
|
else:
|
|
review_matches = bool(proposal.get("reviewed_at")) and has_reviewer
|
|
apply_matches = bool(proposal.get("applied_at")) and has_applier
|
|
return {
|
|
"status_matches": status_matches,
|
|
"review_fields_match": review_matches,
|
|
"apply_fields_match": apply_matches,
|
|
"all": status_matches and review_matches and apply_matches,
|
|
}
|
|
|
|
|
|
def select_expected_proposal(
|
|
*,
|
|
before: dict[str, Any],
|
|
after: dict[str, Any],
|
|
expected_state: str,
|
|
prompt: str,
|
|
reply: str,
|
|
explicit_proposal_id: str | None,
|
|
) -> dict[str, Any]:
|
|
before_by_id = {str(row.get("id")): row for row in before.get("proposals", [])}
|
|
candidates = [row for row in after.get("proposals", []) if row.get("status") == expected_state]
|
|
selected: list[dict[str, Any]] = []
|
|
method = "none"
|
|
|
|
if explicit_proposal_id:
|
|
selected = [row for row in candidates if str(row.get("id")) == explicit_proposal_id]
|
|
method = "explicit_proposal_id"
|
|
else:
|
|
combined_text = f"{prompt}\n{reply}".lower()
|
|
mentioned = {match.group(0).lower() for match in UUID_RE.finditer(combined_text)}
|
|
selected = [row for row in candidates if str(row.get("id", "")).lower() in mentioned]
|
|
if len(selected) == 1:
|
|
method = "prompt_or_reply_uuid"
|
|
else:
|
|
prefix_matches = [
|
|
row
|
|
for row in candidates
|
|
if re.search(rf"\b{re.escape(str(row.get('id', '')).lower()[:8])}\b", combined_text)
|
|
]
|
|
if len(prefix_matches) == 1:
|
|
selected = prefix_matches
|
|
method = "unique_prompt_or_reply_uuid_prefix"
|
|
else:
|
|
changed = [row for row in candidates if before_by_id.get(str(row.get("id"))) != row]
|
|
if len(changed) == 1:
|
|
selected = changed
|
|
method = "unique_new_or_changed_expected_state"
|
|
elif len(candidates) == 1:
|
|
selected = candidates
|
|
method = "unique_expected_state"
|
|
else:
|
|
selected = []
|
|
method = "ambiguous"
|
|
|
|
proposal = selected[0] if len(selected) == 1 else None
|
|
semantics = (
|
|
_state_semantics(proposal, expected_state)
|
|
if proposal
|
|
else {
|
|
"status_matches": False,
|
|
"review_fields_match": False,
|
|
"apply_fields_match": False,
|
|
"all": False,
|
|
}
|
|
)
|
|
return {
|
|
"selection_method": method,
|
|
"candidate_ids": [str(row.get("id")) for row in candidates],
|
|
"selected_proposal": proposal,
|
|
"selected_exactly_one": proposal is not None,
|
|
"state_semantics": semantics,
|
|
}
|
|
|
|
|
|
def validate_private_output_path(requested_output: Path) -> Path:
|
|
requested_output = requested_output.expanduser()
|
|
if requested_output.is_symlink():
|
|
raise CheckpointError("--output must not be a symbolic link")
|
|
try:
|
|
output_parent = requested_output.parent.resolve(strict=True)
|
|
except OSError as exc:
|
|
raise CheckpointError("--output parent must already exist") from exc
|
|
output = output_parent / requested_output.name
|
|
live_profile = LIVE_PROFILE.expanduser().resolve(strict=False)
|
|
if output == live_profile or live_profile in output.parents:
|
|
raise CheckpointError("--output must be outside the live leoclean profile")
|
|
parent_stat = output_parent.stat()
|
|
if not output_parent.is_dir() or parent_stat.st_uid != os.geteuid() or parent_stat.st_mode & 0o077:
|
|
raise CheckpointError("--output parent must be an existing private directory owned by the current user")
|
|
return output
|
|
|
|
|
|
def validate_args(args: argparse.Namespace) -> None:
|
|
if not SAFE_DOCKER_NAME_RE.fullmatch(args.container):
|
|
raise CheckpointError("--container must be an explicit Docker name, not a shell expression")
|
|
if args.container == PRODUCTION_CONTAINER:
|
|
raise CheckpointError("--container must not target the production teleo-pg container")
|
|
if not SAFE_DB_NAME_RE.fullmatch(args.db):
|
|
raise CheckpointError("--db must be an explicit Postgres database name")
|
|
if not args.prompt_id.strip() or not args.prompt.strip():
|
|
raise CheckpointError("--prompt-id and --prompt must be non-empty")
|
|
if args.expected_state not in EXPECTED_STATES:
|
|
raise CheckpointError(f"--expected-state must be one of {', '.join(EXPECTED_STATES)}")
|
|
if args.turn_timeout <= 0:
|
|
raise CheckpointError("--turn-timeout must be positive")
|
|
args.output = validate_private_output_path(args.output)
|
|
if args.proposal_id:
|
|
try:
|
|
args.proposal_id = str(uuid.UUID(args.proposal_id))
|
|
except ValueError as exc:
|
|
raise CheckpointError("--proposal-id must be a UUID") from exc
|
|
|
|
|
|
def _same_service(before: dict[str, Any] | None, after: dict[str, Any] | None) -> bool:
|
|
return bool(before and after and before.get("returncode") == 0 and before == after)
|
|
|
|
|
|
def _reply_mentions_proposal(reply: str, proposal: dict[str, Any] | None) -> bool:
|
|
if not proposal:
|
|
return False
|
|
proposal_id = str(proposal.get("id", "")).lower()
|
|
lowered = reply.lower()
|
|
return bool(proposal_id) and (proposal_id in lowered or proposal_id[:8] in lowered)
|
|
|
|
|
|
def tool_proof_read_exact_proposal(tool_proof: dict[str, Any], proposal: dict[str, Any] | None) -> bool:
|
|
if not proposal:
|
|
return False
|
|
expected = ["show-proposal", str(proposal.get("id") or "")]
|
|
return any(
|
|
invocation.get("argv") == expected
|
|
or (
|
|
invocation.get("argv", [])[:2] == expected
|
|
and invocation.get("argv", [])[2:] in (["--format", "json"], ["--format", "markdown"])
|
|
)
|
|
for invocation in tool_proof.get("invocations", [])
|
|
)
|
|
|
|
|
|
def parse_reply_state_receipt(reply: str) -> dict[str, Any]:
|
|
for line in reversed(reply.splitlines()):
|
|
prefix, separator, payload = line.strip().partition("KB_STATE:")
|
|
if prefix or not separator:
|
|
continue
|
|
try:
|
|
value = json.loads(payload.strip())
|
|
except json.JSONDecodeError as exc:
|
|
return {"valid": False, "error": f"invalid KB_STATE JSON: {exc}"}
|
|
if not isinstance(value, dict):
|
|
return {"valid": False, "error": "KB_STATE must be a JSON object"}
|
|
proposal_id = value.get("proposal_id")
|
|
status = value.get("status")
|
|
applied = value.get("applied")
|
|
try:
|
|
proposal_id = str(uuid.UUID(str(proposal_id)))
|
|
except ValueError:
|
|
return {"valid": False, "error": "KB_STATE proposal_id must be a full UUID"}
|
|
if status not in EXPECTED_STATES or not isinstance(applied, bool):
|
|
return {"valid": False, "error": "KB_STATE status/applied fields are invalid"}
|
|
return {
|
|
"valid": True,
|
|
"proposal_id": proposal_id,
|
|
"status": status,
|
|
"applied": applied,
|
|
}
|
|
return {"valid": False, "error": "reply did not contain a KB_STATE line"}
|
|
|
|
|
|
def reply_state_receipt_matches(receipt: dict[str, Any], proposal: dict[str, Any] | None) -> bool:
|
|
if not receipt.get("valid") or not proposal:
|
|
return False
|
|
expected_applied = bool(proposal.get("applied_at")) and proposal.get("status") == "applied"
|
|
return (
|
|
receipt.get("proposal_id") == str(proposal.get("id"))
|
|
and receipt.get("status") == proposal.get("status")
|
|
and receipt.get("applied") is expected_applied
|
|
)
|
|
|
|
|
|
def write_report(path: Path, report: dict[str, Any], *, secret_values: tuple[str, ...] = ()) -> None:
|
|
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
directory_fd = os.open(path.parent, directory_flags)
|
|
temporary_name = f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
|
|
try:
|
|
payload = (
|
|
json.dumps(redact_value(report, secret_values=secret_values), indent=2, sort_keys=True) + "\n"
|
|
).encode()
|
|
descriptor = os.open(
|
|
temporary_name,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
|
0o600,
|
|
dir_fd=directory_fd,
|
|
)
|
|
try:
|
|
with os.fdopen(descriptor, "wb") as handle:
|
|
handle.write(payload)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
except BaseException:
|
|
try:
|
|
os.close(descriptor)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
os.replace(temporary_name, path.name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd)
|
|
os.chmod(path.name, 0o600, dir_fd=directory_fd, follow_symlinks=False)
|
|
finally:
|
|
try:
|
|
os.unlink(temporary_name, dir_fd=directory_fd)
|
|
except OSError:
|
|
pass
|
|
os.close(directory_fd)
|
|
|
|
|
|
async def run_checkpoint(args: argparse.Namespace) -> dict[str, Any]:
|
|
global _LAST_TERMINATION_SIGNAL
|
|
_LAST_TERMINATION_SIGNAL = None
|
|
validate_args(args)
|
|
report: dict[str, Any] = {
|
|
"schema": "livingip.leoCloneBoundHandlerCheckpoint.v1",
|
|
"generated_at_utc": utc_now(),
|
|
"mode": "live_vps_gatewayrunner_disposable_db_no_send",
|
|
"required_tier": "T2_live_vps_no_send_handler_bound_to_disposable_db",
|
|
"posted_to_telegram": False,
|
|
"orchestrates_review_or_apply": False,
|
|
"changes_live_profile": False,
|
|
"restarts_service": False,
|
|
"production_read_only_invariant_queries": True,
|
|
"target": {"container": args.container, "database": args.db},
|
|
"prompt": {
|
|
"id": args.prompt_id,
|
|
"text": args.prompt,
|
|
"expected_state": args.expected_state,
|
|
"proposal_id": args.proposal_id,
|
|
},
|
|
"source": {
|
|
"platform": "telegram",
|
|
"chat_id": args.chat_id,
|
|
"chat_name": "Leo",
|
|
"chat_type": "group",
|
|
"user_id": args.user_id,
|
|
"user_name": args.user_name,
|
|
},
|
|
"output": str(args.output),
|
|
"errors": [],
|
|
}
|
|
|
|
temp_profile: Path | None = None
|
|
tool_log: Path | None = None
|
|
production_service_before: dict[str, Any] | None = None
|
|
production_counts_before: dict[str, int] | None = None
|
|
production_guard_before: dict[str, Any] | None = None
|
|
target_before: dict[str, Any] | None = None
|
|
target_after: dict[str, Any] | None = None
|
|
target_guard_before: dict[str, Any] | None = None
|
|
target_guard_after: dict[str, Any] | None = None
|
|
target_identity_before: dict[str, Any] | None = None
|
|
target_identity_after: dict[str, Any] | None = None
|
|
target_container_id: str | None = None
|
|
target_read_only_state: dict[str, Any] | None = None
|
|
target_read_only_restore_required = False
|
|
target_read_only_restored: bool | None = None
|
|
gateway: dict[str, Any] = {}
|
|
selection: dict[str, Any] = {}
|
|
bridge: dict[str, Any] = {}
|
|
live_hashes_before: dict[str, str | None] | None = None
|
|
provider_environment: dict[str, str] = {}
|
|
provider_secret_values: tuple[str, ...] = ()
|
|
|
|
try:
|
|
production_service_before = service_state()
|
|
production_guard_before = database_guard_snapshot(PRODUCTION_CONTAINER, PRODUCTION_DB)
|
|
production_counts_before = production_guard_before["counts"]
|
|
live_hashes_before = bridge_hashes()
|
|
target_identity_before = container_identity(args.container)
|
|
production_identity = container_identity(PRODUCTION_CONTAINER)
|
|
isolation_checks = validate_disposable_target_identity(target_identity_before, production_identity)
|
|
target_container_id = str(target_identity_before["id"])
|
|
report["target"].update(
|
|
{
|
|
"requested_container": args.container,
|
|
"bound_container_id": target_container_id,
|
|
"container_identity_before": target_identity_before,
|
|
"isolation_checks": isolation_checks,
|
|
}
|
|
)
|
|
report["production_invariants"] = {
|
|
"container_identity": production_identity,
|
|
"counts_before": production_counts_before,
|
|
"guard_snapshot_before": production_guard_before,
|
|
"service_before": production_service_before,
|
|
}
|
|
|
|
target_read_only_prior = target_read_only_setting(target_container_id, args.db)
|
|
target_read_only_restore_required = not bool(target_read_only_prior.get("already_forced_read_only"))
|
|
report["target"]["read_only_role_restore_obligation_installed"] = target_read_only_restore_required
|
|
target_read_only_state = configure_target_read_only(target_container_id, args.db, enabled=True)
|
|
report["target"]["read_only_role_binding"] = target_read_only_state
|
|
target_guard_before = database_guard_snapshot(target_container_id, args.db)
|
|
if target_guard_before.get("database_identity", {}).get("system_identifier") == production_guard_before.get(
|
|
"database_identity", {}
|
|
).get("system_identifier"):
|
|
raise CheckpointError("target and production resolve to the same Postgres system identifier")
|
|
target_before = target_checkpoint(target_container_id, args.db)
|
|
if target_before.get("database_identity", {}).get("current_database") != args.db:
|
|
raise CheckpointError("target checkpoint did not read the explicitly supplied database")
|
|
report["target_checkpoint_before"] = target_checkpoint_receipt_view(target_before)
|
|
report["target_invariants"] = {"guard_snapshot_before": target_guard_before}
|
|
|
|
temp_profile, copy_audit = copy_profile(run_id=args.run_id, prompt_id=args.prompt_id)
|
|
report["temporary_profile_copy_audit"] = copy_audit
|
|
if args.copy_model_auth:
|
|
report["model_auth_binding"] = copy_ephemeral_model_auth(temp_profile)
|
|
provider_environment, environment_audit = load_ephemeral_provider_environment(temp_profile)
|
|
provider_secret_values = tuple(provider_environment.values())
|
|
report["model_auth_binding"]["environment_binding"] = environment_audit
|
|
else:
|
|
report["model_auth_binding"] = {
|
|
"mode": "excluded",
|
|
"source_contents_recorded": False,
|
|
"source_fingerprint_recorded": False,
|
|
}
|
|
bridge = patch_temp_bridge(temp_profile, target_container_id, args.db)
|
|
tool_log = Path(bridge["tool_log_path"])
|
|
report["temporary_bridge"] = bridge
|
|
gateway = await invoke_gateway_subprocess(
|
|
args,
|
|
temp_profile,
|
|
provider_environment=provider_environment,
|
|
)
|
|
report["gateway"] = gateway
|
|
if gateway.get("handler_error"):
|
|
report["errors"].append(
|
|
{
|
|
"type": "GatewayHandlerError",
|
|
"message": str(gateway["handler_error"]),
|
|
}
|
|
)
|
|
|
|
target_after = target_checkpoint(target_container_id, args.db)
|
|
target_guard_after = database_guard_snapshot(target_container_id, args.db)
|
|
target_identity_after = container_identity(args.container)
|
|
if target_identity_after.get("id") != target_container_id:
|
|
raise CheckpointError("target container name no longer resolves to the bound container ID")
|
|
report["target"]["container_identity_after"] = target_identity_after
|
|
report["target_checkpoint_after"] = target_checkpoint_receipt_view(target_after)
|
|
selection = select_expected_proposal(
|
|
before=target_before,
|
|
after=target_after,
|
|
expected_state=args.expected_state,
|
|
prompt=args.prompt,
|
|
reply=str(gateway.get("reply", "")),
|
|
explicit_proposal_id=args.proposal_id,
|
|
)
|
|
selected = selection.get("selected_proposal")
|
|
identifiers = _uuid_values(selected) if selected else set()
|
|
related_rows = related_canonical_rows(target_container_id, args.db, identifiers)
|
|
report["proposal_checkpoint"] = {
|
|
**selection,
|
|
"selected_proposal": proposal_receipt_view(selected) if selected else None,
|
|
"payload_uuid_values": sorted(identifiers),
|
|
"related_canonical_rows": canonical_rows_receipt_view(related_rows),
|
|
}
|
|
except Exception as exc:
|
|
report["errors"].append({"type": type(exc).__name__, "message": str(exc)})
|
|
report["traceback_tail"] = traceback.format_exc().splitlines()[-12:]
|
|
finally:
|
|
provider_environment.clear()
|
|
if target_before is not None and target_after is None and target_container_id:
|
|
try:
|
|
target_after = target_checkpoint(target_container_id, args.db)
|
|
report["target_checkpoint_after"] = target_checkpoint_receipt_view(target_after)
|
|
except Exception as exc:
|
|
report["errors"].append({"type": type(exc).__name__, "message": f"target postflight: {exc}"})
|
|
if target_guard_before is not None and target_guard_after is None and target_container_id:
|
|
try:
|
|
target_guard_after = database_guard_snapshot(target_container_id, args.db)
|
|
except Exception as exc:
|
|
report["errors"].append({"type": type(exc).__name__, "message": f"target guard postflight: {exc}"})
|
|
if target_identity_before is not None and target_identity_after is None:
|
|
try:
|
|
target_identity_after = container_identity(args.container)
|
|
report["target"]["container_identity_after"] = target_identity_after
|
|
except Exception as exc:
|
|
report["errors"].append({"type": type(exc).__name__, "message": f"target identity postflight: {exc}"})
|
|
|
|
if target_container_id and (target_read_only_state is not None or target_read_only_restore_required):
|
|
try:
|
|
if target_read_only_restore_required:
|
|
reset = configure_target_read_only(target_container_id, args.db, enabled=False)
|
|
target_read_only_restored = reset.get("verified_transaction_read_only") == "off"
|
|
else:
|
|
target_read_only_restored = True
|
|
except Exception as exc:
|
|
target_read_only_restored = False
|
|
report["errors"].append({"type": type(exc).__name__, "message": f"target read-only reset: {exc}"})
|
|
report.setdefault("target", {})["read_only_role_restored"] = target_read_only_restored
|
|
|
|
try:
|
|
if not bridge or not target_container_id or not target_before:
|
|
raise CheckpointError("tool proof prerequisites were not established")
|
|
tool_proof = read_tool_proof(
|
|
tool_log,
|
|
target_container_id,
|
|
args.db,
|
|
run_nonce=str(bridge["run_nonce"]),
|
|
database_identity=target_before["database_identity"],
|
|
)
|
|
except Exception as exc:
|
|
report["errors"].append({"type": type(exc).__name__, "message": f"tool proof readback: {exc}"})
|
|
tool_proof = {
|
|
"parse_errors": [str(exc)],
|
|
"event_count": 0,
|
|
"duplicate_invocation_ids": [],
|
|
"complete_start_end_pairing": False,
|
|
"invocations": [],
|
|
"invocation_count": 0,
|
|
"all_bound_to_supplied_target": False,
|
|
"all_completed_successfully": False,
|
|
}
|
|
report["tool_proof"] = tool_proof
|
|
temp_root = temp_profile.parent if temp_profile else None
|
|
child_process = gateway.get("child_process") or {}
|
|
child_absent = not _ACTIVE_GATEWAY_CHILDREN and (
|
|
not child_process
|
|
or (
|
|
child_process.get("alive_after_readback") is False
|
|
and child_process.get("process_group_alive_after_readback") is False
|
|
)
|
|
)
|
|
if child_absent:
|
|
temp_removed = remove_tree(temp_root)
|
|
else:
|
|
temp_removed = False
|
|
report["errors"].append(
|
|
{"type": "CleanupSafetyError", "message": "temporary profile retained because gateway group is alive"}
|
|
)
|
|
absence_verified = temp_path_absent(temp_root)
|
|
report["cleanup"] = {
|
|
"temp_root": str(temp_root) if temp_root else None,
|
|
"temp_profile_removed": temp_removed,
|
|
"temp_session_removed_with_profile": temp_removed,
|
|
"absence_verified_independently": absence_verified,
|
|
"active_temp_root_registry_clear": temp_root not in _ACTIVE_TEMP_ROOTS if temp_root else True,
|
|
"termination_signal": _LAST_TERMINATION_SIGNAL,
|
|
"gateway_group_confirmed_absent_before_profile_removal": child_absent,
|
|
}
|
|
|
|
try:
|
|
production_guard_after = database_guard_snapshot(PRODUCTION_CONTAINER, PRODUCTION_DB)
|
|
production_counts_after = production_guard_after["counts"]
|
|
except Exception as exc:
|
|
production_guard_after = None
|
|
production_counts_after = None
|
|
report["errors"].append({"type": type(exc).__name__, "message": f"production count postflight: {exc}"})
|
|
try:
|
|
production_service_after = service_state()
|
|
except Exception as exc: # service_state currently returns errors instead of raising
|
|
production_service_after = None
|
|
report["errors"].append({"type": type(exc).__name__, "message": f"service postflight: {exc}"})
|
|
live_hashes_after = bridge_hashes()
|
|
|
|
production = report.setdefault("production_invariants", {})
|
|
production.update(
|
|
{
|
|
"counts_before": production_counts_before,
|
|
"counts_after": production_counts_after,
|
|
"counts_unchanged": production_counts_before is not None
|
|
and production_counts_before == production_counts_after,
|
|
"guard_snapshot_before": production_guard_before,
|
|
"guard_snapshot_after": production_guard_after,
|
|
"guard_snapshot_unchanged": production_guard_before is not None
|
|
and production_guard_before == production_guard_after,
|
|
"service_before": production_service_before,
|
|
"service_after": production_service_after,
|
|
"service_unchanged": _same_service(production_service_before, production_service_after),
|
|
"live_bridge_hashes_before": live_hashes_before,
|
|
"live_bridge_hashes_after": live_hashes_after,
|
|
"live_bridge_unchanged": live_hashes_before is not None and live_hashes_before == live_hashes_after,
|
|
}
|
|
)
|
|
|
|
target_invariants = report.setdefault("target_invariants", {})
|
|
target_invariants.update(
|
|
{
|
|
"guard_snapshot_before": target_guard_before,
|
|
"guard_snapshot_after": target_guard_after,
|
|
"guard_snapshot_unchanged": target_guard_before is not None
|
|
and target_guard_before == target_guard_after,
|
|
"container_identity_before": target_identity_before,
|
|
"container_identity_after": target_identity_after,
|
|
"container_identity_unchanged": target_identity_before is not None
|
|
and target_identity_before == target_identity_after,
|
|
"postgres_system_distinct_from_production": bool(target_guard_before and production_guard_before)
|
|
and target_guard_before.get("database_identity", {}).get("system_identifier")
|
|
!= production_guard_before.get("database_identity", {}).get("system_identifier"),
|
|
"read_only_during_checkpoint": bool(target_guard_before and target_guard_after)
|
|
and target_guard_before.get("database_identity", {}).get("transaction_read_only") == "on"
|
|
and target_guard_after.get("database_identity", {}).get("transaction_read_only") == "on",
|
|
"read_only_role_restored": target_read_only_restored is True,
|
|
}
|
|
)
|
|
|
|
selected = selection.get("selected_proposal") if selection else None
|
|
reply = str(gateway.get("reply", ""))
|
|
tool_surface = gateway.get("tool_surface") or {}
|
|
allowed_tool_names = set(tool_surface.get("allowed_tools") or [])
|
|
transcript_calls = [
|
|
event.get("tool_name")
|
|
for event in (gateway.get("transcript_tool_trace") or {}).get("events", [])
|
|
if event.get("phase") == "call"
|
|
]
|
|
terminal_attempts = gateway.get("transcript_terminal_attempts") or transcript_terminal_attempt_summary(
|
|
(gateway.get("transcript_tool_trace") or {}).get("events", [])
|
|
)
|
|
gateway["transcript_terminal_attempts"] = terminal_attempts
|
|
child_process = gateway.get("child_process") or {}
|
|
reply_state_receipt = parse_reply_state_receipt(reply)
|
|
gateway["reply_state_receipt"] = reply_state_receipt
|
|
checks = {
|
|
"target_is_not_production_container": target_invariants.get("postgres_system_distinct_from_production")
|
|
is True,
|
|
"target_container_identity_unchanged": target_invariants.get("container_identity_unchanged") is True,
|
|
"target_isolation_contract_passed": bool(report.get("target", {}).get("isolation_checks"))
|
|
and all(report.get("target", {}).get("isolation_checks", {}).values()),
|
|
"target_database_matches_supplied_db": bool(target_after)
|
|
and target_after.get("database_identity", {}).get("current_database") == args.db,
|
|
"target_row_content_unchanged": target_invariants.get("guard_snapshot_unchanged") is True,
|
|
"target_forced_read_only_during_checkpoint": target_invariants.get("read_only_during_checkpoint") is True,
|
|
"target_read_only_role_restored": target_invariants.get("read_only_role_restored") is True,
|
|
"telegram_shaped_source_authorized": gateway.get("authorized") is True,
|
|
"real_gateway_handler_invoked": gateway.get("handler_invoked") is True,
|
|
"machine_readable_reply_nonempty": bool(reply.strip()),
|
|
"model_free_fallback_not_used": gateway.get("model_free_fallback_used") is False,
|
|
"kb_tool_call_observed": tool_proof["invocation_count"] > 0,
|
|
"all_kb_tool_calls_bound_to_supplied_target": tool_proof["all_bound_to_supplied_target"],
|
|
"all_kb_tool_calls_completed_successfully": tool_proof["all_completed_successfully"],
|
|
"selected_exactly_one_expected_proposal": selection.get("selected_exactly_one") is True,
|
|
"expected_state_semantics_match": selection.get("state_semantics", {}).get("all") is True,
|
|
"reply_mentions_selected_proposal": _reply_mentions_proposal(reply, selected),
|
|
"tool_receipt_read_exact_selected_proposal": tool_proof_read_exact_proposal(tool_proof, selected),
|
|
"reply_state_receipt_valid": reply_state_receipt.get("valid") is True,
|
|
"reply_state_receipt_matches_selected_row": reply_state_receipt_matches(reply_state_receipt, selected),
|
|
"production_counts_unchanged": production.get("counts_unchanged") is True,
|
|
"production_guard_snapshot_unchanged": production.get("guard_snapshot_unchanged") is True,
|
|
"gateway_service_unchanged": production.get("service_unchanged") is True,
|
|
"live_bridge_unchanged": production.get("live_bridge_unchanged") is True,
|
|
"send_message_tool_not_exposed": tool_surface.get("send_message_tool_enabled") is False,
|
|
"actual_registry_tools_exactly_allowlisted": allowed_tool_names == {"skills_list", "skill_view", "terminal"}
|
|
and set(tool_surface.get("actual_registry_tools") or []) == allowed_tool_names,
|
|
"gateway_has_no_delivery_adapters": tool_surface.get("gateway_adapters_verified_mapping") is True
|
|
and tool_surface.get("gateway_adapter_count") == 0,
|
|
"terminal_restricted_to_clone_wrapper": (tool_surface.get("terminal_restricted_to_clone_wrapper") is True),
|
|
"terminal_subprocess_has_no_provider_credentials": (
|
|
tool_surface.get("terminal_subprocess_inherits_provider_credentials") is False
|
|
),
|
|
"terminal_command_rejections_absent": terminal_attempts.get("rejected_call_count") == 0,
|
|
"terminal_nonzero_results_absent": terminal_attempts.get("nonzero_call_count") == 0,
|
|
"terminal_call_results_complete_and_receipted": terminal_attempts.get("result_code_unknown_call_count") == 0
|
|
and terminal_attempts.get("call_count") == tool_proof.get("invocation_count"),
|
|
"all_transcript_tool_calls_within_allowlist": bool(transcript_calls)
|
|
and all(name in allowed_tool_names for name in transcript_calls),
|
|
"gateway_child_and_descendants_absent": (
|
|
child_process.get("alive_after_readback") is False
|
|
and child_process.get("process_group_alive_after_readback") is False
|
|
),
|
|
"gateway_child_ready_in_own_process_group": child_process.get("readiness_verified") is True,
|
|
"gateway_child_completed_without_timeout_or_forced_kill": child_process.get("timed_out") is False
|
|
and child_process.get("exitcode") == 0
|
|
and (child_process.get("termination") or {}).get("attempted") is False,
|
|
"gateway_result_used_private_file": child_process.get("result_transport") == "private_temp_file",
|
|
"temp_profile_removed": temp_removed,
|
|
"temp_session_removed": temp_removed,
|
|
"temp_root_absence_verified_independently": absence_verified,
|
|
"temp_root_unregistered": report["cleanup"]["active_temp_root_registry_clear"],
|
|
"posted_to_telegram_false": (
|
|
report["posted_to_telegram"] is False
|
|
and gateway.get("posted_to_telegram") is False
|
|
and tool_surface.get("send_message_tool_enabled") is False
|
|
and tool_surface.get("gateway_adapter_count") == 0
|
|
),
|
|
"review_apply_not_orchestrated": report["orchestrates_review_or_apply"] is False
|
|
and target_invariants.get("guard_snapshot_unchanged") is True,
|
|
}
|
|
report["checks"] = checks
|
|
report["status"] = "pass" if not report["errors"] and all(checks.values()) else "fail"
|
|
report["claim_ceiling"] = (
|
|
"T2 live VPS GatewayRunner checkpoint with a KB-only tool allowlist, a terminal handler restricted "
|
|
"to read-only commands on the supplied disposable Postgres target, no delivery adapters, and no "
|
|
"send_message tool. It reads production invariants before/after but does not review, approve, apply, "
|
|
"restart, post, or expose a production mutation tool to the model."
|
|
)
|
|
report["completed_at_utc"] = utc_now()
|
|
report = redact_value(report, secret_values=provider_secret_values)
|
|
write_report(args.output, report, secret_values=provider_secret_values)
|
|
return report
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--container", required=True, help="Disposable Postgres Docker container name.")
|
|
parser.add_argument("--db", required=True, help="Database inside the disposable container.")
|
|
parser.add_argument("--prompt-id", required=True, help="Stable checkpoint prompt identifier.")
|
|
parser.add_argument("--prompt", required=True, help="Exact operator prompt passed to GatewayRunner.")
|
|
parser.add_argument("--expected-state", required=True, choices=EXPECTED_STATES)
|
|
parser.add_argument("--output", required=True, type=Path, help="Machine-readable JSON receipt path.")
|
|
parser.add_argument(
|
|
"--proposal-id",
|
|
help="Optional exact proposal UUID. Otherwise selection uses prompt/reply UUIDs or an unambiguous state delta.",
|
|
)
|
|
parser.add_argument("--chat-id", default=DEFAULT_CHAT_ID)
|
|
parser.add_argument("--user-id", default=DEFAULT_USER_ID)
|
|
parser.add_argument("--user-name", default=DEFAULT_USER_NAME)
|
|
parser.add_argument("--run-id", default=datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S"))
|
|
parser.add_argument("--turn-timeout", type=int, default=300)
|
|
parser.add_argument(
|
|
"--copy-model-auth",
|
|
action="store_true",
|
|
help=(
|
|
"Copy only the live profile auth.json into the UUID-scoped temporary "
|
|
"profile. The copy may refresh independently and is deleted with the "
|
|
"temporary profile; credential contents are never retained in the receipt."
|
|
),
|
|
)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
try:
|
|
with termination_cleanup_handlers():
|
|
report = asyncio.run(run_checkpoint(args))
|
|
except CheckpointError as exc:
|
|
print(json.dumps({"status": "rejected", "error": str(exc)}, sort_keys=True))
|
|
return 2
|
|
except TerminationRequested as exc:
|
|
print(json.dumps({"status": "terminated", "signal": exc.signum, "output": str(args.output)}, sort_keys=True))
|
|
return 128 + exc.signum
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": report["status"],
|
|
"output": str(args.output),
|
|
"checks_passed": sum(1 for value in report.get("checks", {}).values() if value is True),
|
|
"checks_total": len(report.get("checks", {})),
|
|
"error_count": len(report.get("errors", [])),
|
|
},
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0 if report["status"] == "pass" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|