teleo-infrastructure/scripts/collect_telegram_visible_unseen_chain_state.py

616 lines
25 KiB
Python

#!/usr/bin/env python3
"""Collect fail-closed read-only state for the unseen Telegram chain."""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import shlex
import subprocess
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
try:
import build_telegram_visible_unseen_chain_packet as packet_builder
import verify_leo_unseen_reasoning_chain as unseen
except ImportError: # pragma: no cover - imported as scripts.* in tests
from scripts import build_telegram_visible_unseen_chain_packet as packet_builder
from scripts import verify_leo_unseen_reasoning_chain as unseen
SCHEMA = "livingip.telegramVisibleUnseenChainState.v1"
REPORT_DIR = Path("docs/reports/leo-working-state-20260709")
DEFAULT_PACKET = packet_builder.DEFAULT_OUT
DEFAULT_PREFLIGHT_OUT = REPORT_DIR / "telegram-visible-unseen-chain-preflight-current.json"
DEFAULT_PREFLIGHT_MARKDOWN_OUT = REPORT_DIR / "telegram-visible-unseen-chain-preflight-current.md"
DEFAULT_POSTFLIGHT_OUT = REPORT_DIR / "telegram-visible-unseen-chain-postflight-current.json"
DEFAULT_POSTFLIGHT_MARKDOWN_OUT = REPORT_DIR / "telegram-visible-unseen-chain-postflight-current.md"
DEFAULT_SSH_TARGET = "root@77.42.65.182"
DEFAULT_SSH_KEY = Path.home() / ".ssh/livingip_hetzner_20260604_ed25519"
DEFAULT_MANIFEST_SQL = Path("ops/postgres_parity_manifest.sql")
REMOTE_REPO = "/opt/teleo-eval/workspaces/deploy-infra"
SUBJECT_REF_RE = re.compile(
r"^(?:[0-9a-f]{8}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$",
re.IGNORECASE,
)
SHA40_RE = re.compile(r"^[0-9a-f]{40}$")
@dataclass(frozen=True)
class CommandResult:
returncode: int
stdout: str
stderr: str
Runner = Callable[[list[str], str, int], CommandResult]
def subprocess_runner(command: list[str], input_text: str, timeout: int) -> CommandResult:
try:
proc = subprocess.run(
command,
input=input_text,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
except subprocess.TimeoutExpired as exc:
return CommandResult(124, str(exc.stdout or ""), f"command timed out after {timeout}s")
return CommandResult(proc.returncode, proc.stdout, proc.stderr)
def _load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def _canonical_sha256(value: Any) -> str:
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _sha256_file(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def parse_key_value_lines(text: str) -> dict[str, str]:
values: dict[str, str] = {}
for line in text.splitlines():
if "=" not in line:
continue
key, value = line.split("=", 1)
values[key] = value.strip()
return values
def parse_manifest_output(raw: str) -> dict[str, Any]:
rows = []
try:
for line in raw.splitlines():
candidate = line.strip()
if not candidate or candidate in {"BEGIN", "SET", "ROLLBACK"}:
continue
value = json.loads(candidate)
if isinstance(value, dict):
rows.append(value)
except (TypeError, json.JSONDecodeError) as exc:
return {"status": "error", "error": f"manifest_parse_error:{type(exc).__name__}"}
tables = {
f"{row['schema']}.{row['table']}": {
"row_count": int(row["row_count"]),
"rowset_md5": row["rowset_md5"],
}
for row in rows
if row.get("kind") == "table"
}
singleton_kinds = {
"schemas",
"extensions",
"application_roles",
"columns",
"constraints",
"indexes",
"sequences",
"views",
"functions",
"triggers",
"types",
"policies",
}
singleton = {str(row["kind"]): row.get("items", []) for row in rows if row.get("kind") in singleton_kinds}
identity_rows = [row for row in rows if row.get("kind") == "identity"]
if len(identity_rows) != 1 or not tables or singleton_kinds - set(singleton):
return {"status": "error", "error": "canonical_database_fingerprint_manifest_incomplete"}
identity = identity_rows[0]
stable = {
"identity": {
key: identity.get(key)
for key in (
"database",
"server_version_num",
"server_encoding",
"database_collation",
"database_ctype",
"transaction_read_only",
)
},
"singleton_sha256": {key: _canonical_sha256(value) for key, value in sorted(singleton.items())},
"tables": tables,
}
return {
"schema": "livingip.teleoCanonicalDatabaseFingerprint.v1",
"status": "ok",
"fingerprint_sha256": _canonical_sha256(stable),
"table_count": len(tables),
"total_rows": sum(item["row_count"] for item in tables.values()),
"table_rows_sha256": _canonical_sha256(tables),
"structure_sha256": _canonical_sha256(stable["singleton_sha256"]),
"transaction_read_only": identity.get("transaction_read_only"),
}
def packet_checks(packet: dict[str, Any]) -> dict[str, bool]:
target = packet.get("target") if isinstance(packet.get("target"), dict) else {}
exact = packet_builder.exact_messages()
expected_tooling = packet_builder.tooling_binding()
return {
"packet_schema_supported": packet.get("schema") == packet_builder.SCHEMA,
"packet_ready_for_authorization_request": packet.get("ready_to_request_action_time_authorization") is True,
"packet_not_sent": packet.get("telegram_visible_messages_sent") is False,
"packet_has_no_authorization_recorded": packet.get("telegram_visible_message_authorization_present") is False,
"packet_does_not_mutate_db": packet.get("mutates_db") is False,
"packet_targets_exact_leo_chat": target.get("chat_label") == packet_builder.DEFAULT_CHAT_LABEL
and target.get("chat_id") == packet_builder.DEFAULT_CHAT_ID,
"packet_requires_authenticated_chrome": target.get("allowed_transport") == "authenticated_chrome_telegram_ui",
"packet_prompt_ids_exact": target.get("expected_prompt_ids") == [item["id"] for item in unseen.PROMPTS],
"packet_messages_exact": [item.get("message") for item in packet.get("exact_messages", [])]
== [item["message"] for item in exact],
"packet_protocol_sha256_present": isinstance(packet.get("protocol_sha256"), str)
and len(packet["protocol_sha256"]) == 64,
"packet_tooling_hashes_match_current_sources": packet.get("tooling_binding") == expected_tooling,
}
def ssh_command(args: argparse.Namespace, remote_command: str) -> list[str]:
return [
"ssh",
"-i",
str(args.ssh_key),
"-o",
"BatchMode=yes",
"-o",
"StrictHostKeyChecking=accept-new",
"-o",
f"ConnectTimeout={args.connect_timeout}",
args.ssh_target,
remote_command,
]
def deploy_readback_command() -> str:
repo = shlex.quote(REMOTE_REPO)
return (
f"cd {repo} && "
"deploy_head=$(git rev-parse HEAD) && "
"last_deploy_sha=$(cat /opt/teleo-eval/.last-deploy-sha) && "
'printf \'deploy_head=%s\\nlast_deploy_sha=%s\\n\' "$deploy_head" "$last_deploy_sha" && '
'if git merge-base --is-ancestor "$last_deploy_sha" "$deploy_head"; then '
"printf 'deploy_stamp_ancestor=true\\n'; else printf 'deploy_stamp_ancestor=false\\n'; fi"
)
def service_readback_command() -> str:
return (
"systemctl show leoclean-gateway.service "
"-p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp --no-pager"
)
def fingerprint_readback_command() -> str:
return "docker exec -i teleo-pg psql -U postgres -d teleo -At -q -v ON_ERROR_STOP=1"
def subject_readback_sql(subject_ref: str) -> str:
if not SUBJECT_REF_RE.fullmatch(subject_ref):
raise ValueError("subject_ref must be an eight-hex prefix or UUID")
prefix = subject_ref.casefold()
return f"""
\\set ON_ERROR_STOP on
begin transaction isolation level repeatable read read only;
select jsonb_build_object(
'subject_ref', '{prefix}',
'matches', coalesce(jsonb_agg(item order by item->>'table', item->'row'->>'id'), '[]'::jsonb)
)::text
from (
select jsonb_build_object(
'table', 'public.claims',
'row', to_jsonb(claim_row),
'evidence_count', (select count(*) from public.claim_evidence evidence where evidence.claim_id = claim_row.id)
) as item
from public.claims claim_row
where lower(claim_row.id::text) like '{prefix}%'
union all
select jsonb_build_object(
'table', 'public.beliefs',
'row', to_jsonb(belief_row),
'evidence_count', null
) as item
from public.beliefs belief_row
where lower(belief_row.id::text) like '{prefix}%'
) matched;
rollback;
""".strip()
def _parse_single_json_line(raw: str) -> dict[str, Any] | None:
for line in raw.splitlines():
candidate = line.strip()
if not candidate or candidate in {"BEGIN", "SET", "ROLLBACK"}:
continue
try:
value = json.loads(candidate)
except json.JSONDecodeError:
continue
if isinstance(value, dict):
return value
return None
def collect_remote(args: argparse.Namespace, runner: Runner) -> tuple[dict[str, Any], int, str]:
manifest_sql = args.manifest_sql.read_text(encoding="utf-8")
calls: list[tuple[str, list[str], str, int]] = [
("deploy", ssh_command(args, deploy_readback_command()), "", args.timeout),
("service_before", ssh_command(args, service_readback_command()), "", args.timeout),
(
"fingerprint_before",
ssh_command(args, fingerprint_readback_command()),
manifest_sql,
args.fingerprint_timeout,
),
(
"fingerprint_after",
ssh_command(args, fingerprint_readback_command()),
manifest_sql,
args.fingerprint_timeout,
),
("service_after", ssh_command(args, service_readback_command()), "", args.timeout),
]
if args.subject_ref:
calls.append(
(
"subject",
ssh_command(args, fingerprint_readback_command()),
subject_readback_sql(args.subject_ref),
args.timeout,
)
)
results: dict[str, CommandResult] = {}
attempts: dict[str, int] = {}
for name, command, input_text, timeout in calls:
result = CommandResult(1, "", "not attempted")
for attempt in range(1, max(1, args.ssh_attempts) + 1):
result = runner(command, input_text, timeout)
attempts[name] = attempt
if result.returncode == 0:
break
results[name] = result
deploy_values = parse_key_value_lines(results["deploy"].stdout)
service_before = parse_key_value_lines(results["service_before"].stdout)
service_after = parse_key_value_lines(results["service_after"].stdout)
fingerprint_before = parse_manifest_output(results["fingerprint_before"].stdout)
fingerprint_after = parse_manifest_output(results["fingerprint_after"].stdout)
subject = _parse_single_json_line(results["subject"].stdout) if "subject" in results else None
returncode = next((result.returncode for result in results.values() if result.returncode != 0), 0)
stderr = "\n".join(f"{name}: {result.stderr.strip()}" for name, result in results.items() if result.stderr.strip())
return (
{
"deploy_head": deploy_values.get("deploy_head", ""),
"last_deploy_sha": deploy_values.get("last_deploy_sha", ""),
"deploy_stamp_ancestor": deploy_values.get("deploy_stamp_ancestor") == "true",
"workspace_head_matches_deployed_stamp": deploy_values.get("deploy_head")
== deploy_values.get("last_deploy_sha"),
"service_before": service_before,
"service_after": service_after,
"fingerprint_before": fingerprint_before,
"fingerprint_after": fingerprint_after,
"subject_readback": subject,
"readback_attempts": attempts,
},
returncode,
stderr,
)
def _service_active(state: dict[str, Any]) -> bool:
return state.get("ActiveState") == "active" and state.get("SubState") == "running"
def _service_identity(state: dict[str, Any]) -> dict[str, Any]:
return {key: state.get(key) for key in ("MainPID", "NRestarts", "ExecMainStartTimestamp")}
def _remote_checks(
remote: dict[str, Any],
*,
returncode: int,
subject_required: bool,
expected_subject_ref: str | None,
) -> dict[str, bool]:
before = remote.get("fingerprint_before") or {}
after = remote.get("fingerprint_after") or {}
service_before = remote.get("service_before") or {}
service_after = remote.get("service_after") or {}
subject = remote.get("subject_readback") or {}
matches = subject.get("matches") if isinstance(subject, dict) else None
checks = {
"ssh_readbacks_succeeded": returncode == 0,
"workspace_head_is_sha": bool(SHA40_RE.fullmatch(str(remote.get("deploy_head") or ""))),
"deployed_stamp_is_sha": bool(SHA40_RE.fullmatch(str(remote.get("last_deploy_sha") or ""))),
"deployed_stamp_is_ancestor_of_workspace_head": remote.get("deploy_stamp_ancestor") is True,
"service_active_before_and_after": _service_active(service_before) and _service_active(service_after),
"service_identity_complete": all(
value
for value in (
service_before.get("MainPID"),
service_before.get("NRestarts"),
service_before.get("ExecMainStartTimestamp"),
service_after.get("MainPID"),
service_after.get("NRestarts"),
service_after.get("ExecMainStartTimestamp"),
)
),
"service_process_unchanged_during_probe": _service_identity(service_before) == _service_identity(service_after),
"first_fingerprint_complete": before.get("status") == "ok" and int(before.get("table_count") or 0) > 0,
"second_fingerprint_complete": after.get("status") == "ok" and int(after.get("table_count") or 0) > 0,
"fingerprints_are_read_only": before.get("transaction_read_only") == "on"
and after.get("transaction_read_only") == "on",
"canonical_fingerprint_unchanged_during_probe": bool(before.get("fingerprint_sha256"))
and before.get("fingerprint_sha256") == after.get("fingerprint_sha256"),
}
if subject_required:
checks.update(
{
"subject_readback_present": isinstance(subject, dict),
"subject_reference_matches_request": subject.get("subject_ref")
== str(expected_subject_ref or "").casefold(),
"subject_resolved_to_exactly_one_db_object": isinstance(matches, list) and len(matches) == 1,
}
)
return checks
def collect_state(args: argparse.Namespace, *, runner: Runner = subprocess_runner) -> dict[str, Any]:
packet = _load_json(args.packet)
local_packet_checks = packet_checks(packet)
remote, remote_returncode, remote_stderr = collect_remote(args, runner)
remote_checks = _remote_checks(
remote,
returncode=remote_returncode,
subject_required=args.phase == "postflight",
expected_subject_ref=args.subject_ref,
)
baseline = _load_json(args.baseline) if args.baseline else None
baseline_checks: dict[str, bool] = {}
if args.phase == "postflight":
baseline_checks = {
"baseline_supplied": baseline is not None,
"baseline_is_passing_preflight": bool(
baseline and baseline.get("phase") == "preflight" and baseline.get("status") == "pass"
),
"baseline_protocol_matches_packet": bool(
baseline and baseline.get("protocol_sha256") == packet.get("protocol_sha256")
),
"baseline_packet_file_matches": bool(
baseline and baseline.get("packet_file_sha256") == _sha256_file(args.packet)
),
"canonical_fingerprint_matches_preflight": bool(
baseline
and (baseline.get("remote") or {}).get("fingerprint_after", {}).get("fingerprint_sha256")
== remote.get("fingerprint_after", {}).get("fingerprint_sha256")
),
"gateway_process_matches_preflight": bool(
baseline
and _service_identity((baseline.get("remote") or {}).get("service_after") or {})
== _service_identity(remote.get("service_after") or {})
),
}
status = (
"pass"
if all(local_packet_checks.values()) and all(remote_checks.values()) and all(baseline_checks.values())
else "fail"
)
packet_file_sha256 = _sha256_file(args.packet)
state_token = _canonical_sha256(
{
"phase": args.phase,
"packet_file_sha256": packet_file_sha256,
"protocol_sha256": packet.get("protocol_sha256"),
"deploy_head": remote.get("deploy_head"),
"service": _service_identity(remote.get("service_after") or {}),
"fingerprint": remote.get("fingerprint_after"),
"subject_readback": remote.get("subject_readback"),
}
)
exact_gate = ""
if status != "pass":
failed = [
key
for checks in (local_packet_checks, remote_checks, baseline_checks)
for key, value in checks.items()
if value is not True
]
exact_gate = remote_stderr.strip() or ", ".join(failed)
current_canary = (
"Read-only pre-send readiness for the exact three-turn unseen Leo chain in Telegram"
if args.phase == "preflight"
else "Read-only post-send DB grounding and unchanged-state proof for the exact Telegram chain"
)
return {
"schema": SCHEMA,
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
"mode": "telegram_visible_unseen_chain_zero_mutation_state_readback",
"phase": args.phase,
"status": status,
"ready_for_action_time_authorization": args.phase == "preflight" and status == "pass",
"ready_for_visible_capture_verification": args.phase == "postflight" and status == "pass",
"telegram_visible_messages_sent_by_collector": False,
"production_apply_executed": False,
"mutates_db": False,
"packet_path": str(args.packet),
"packet_file_sha256": packet_file_sha256,
"protocol_sha256": packet.get("protocol_sha256"),
"baseline_path": str(args.baseline) if args.baseline else "",
"subject_ref": args.subject_ref or "",
"state_token_sha256": state_token,
"packet_checks": local_packet_checks,
"remote_checks": remote_checks,
"baseline_checks": baseline_checks,
"remote": remote,
"remote_returncode": remote_returncode,
"remote_stderr": remote_stderr,
"blocker_readback": {
"current_canary": current_canary,
"attempted_routes": [
"local exact packet validation",
"SSH deploy stamp and gateway service readback",
"two canonical repeatable-read read-only Postgres parity manifests",
*(["independent selected-object DB readback"] if args.subject_ref else []),
],
"exact_gate": exact_gate,
"clear_CTA": (
"No user action is needed; repair the named readback check and rerun this collector."
if exact_gate
else "Use the packet's exact authorization sentence only when ready to cross the send boundary."
),
"next_non_user_action": (
"Run the authenticated-Chrome three-turn flow only after exact authorization, then collect postflight."
if args.phase == "preflight" and status == "pass"
else "Repair the failed preflight check and rerun before requesting authorization."
if args.phase == "preflight"
else "Run the visible capture verifier against this postflight and the retained Chrome evidence."
),
},
"claim_ceiling": {
"current_tier": "T3_live_readonly" if status == "pass" else "T0_failed_preflight",
"proven": (
"Two canonical read-only snapshots and the gateway process were unchanged during this probe."
if status == "pass"
else "No live readiness claim; one or more fail-closed checks did not pass."
),
"not_proven": [
"Telegram-visible delivery or reply behavior",
"authorization to send Telegram messages",
"canonical mutation",
],
},
}
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def write_markdown(path: Path, data: dict[str, Any]) -> None:
remote = data.get("remote") or {}
service = remote.get("service_after") or {}
fingerprint = remote.get("fingerprint_after") or {}
lines = [
f"# Telegram-Visible Unseen Chain {data['phase'].title()}",
"",
f"Generated UTC: `{data['generated_at_utc']}`",
f"Status: `{data['status']}`",
f"Protocol SHA256: `{data['protocol_sha256']}`",
f"Packet file SHA256: `{data['packet_file_sha256']}`",
f"State token SHA256: `{data['state_token_sha256']}`",
f"Messages sent by collector: `{data['telegram_visible_messages_sent_by_collector']}`",
f"Mutates DB: `{data['mutates_db']}`",
"",
"## Checks",
"",
]
for section in ("packet_checks", "remote_checks", "baseline_checks"):
for key, value in sorted(data.get(section, {}).items()):
lines.append(f"- `{key}`: `{value}`")
lines.extend(
[
"",
"## Live Readback",
"",
f"- Deploy head: `{remote.get('deploy_head', '')}`",
f"- Deploy stamp: `{remote.get('last_deploy_sha', '')}`",
f"- Gateway MainPID: `{service.get('MainPID', '')}`",
f"- Gateway NRestarts: `{service.get('NRestarts', '')}`",
f"- Canonical fingerprint: `{fingerprint.get('fingerprint_sha256', '')}`",
f"- Canonical tables: `{fingerprint.get('table_count', '')}`",
f"- Canonical rows: `{fingerprint.get('total_rows', '')}`",
f"- Selected object ref: `{data.get('subject_ref', '')}`",
"",
"## Claim Ceiling",
"",
data["claim_ceiling"]["proven"],
"",
]
)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(lines), encoding="utf-8")
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--phase", choices=("preflight", "postflight"), default="preflight")
parser.add_argument("--packet", type=Path, default=DEFAULT_PACKET)
parser.add_argument("--baseline", type=Path)
parser.add_argument("--subject-ref")
parser.add_argument("--manifest-sql", type=Path, default=DEFAULT_MANIFEST_SQL)
parser.add_argument("--ssh-target", default=DEFAULT_SSH_TARGET)
parser.add_argument("--ssh-key", type=Path, default=DEFAULT_SSH_KEY)
parser.add_argument("--connect-timeout", type=int, default=10)
parser.add_argument("--timeout", type=int, default=60)
parser.add_argument("--fingerprint-timeout", type=int, default=240)
parser.add_argument("--ssh-attempts", type=int, default=3)
parser.add_argument("--out", type=Path)
parser.add_argument("--markdown-out", type=Path)
args = parser.parse_args(argv)
if args.phase == "postflight" and (not args.baseline or not args.subject_ref):
parser.error("postflight requires --baseline and --subject-ref")
if args.subject_ref and not SUBJECT_REF_RE.fullmatch(args.subject_ref):
parser.error("--subject-ref must be an eight-hex prefix or UUID")
if args.out is None:
args.out = DEFAULT_PREFLIGHT_OUT if args.phase == "preflight" else DEFAULT_POSTFLIGHT_OUT
if args.markdown_out is None:
args.markdown_out = (
DEFAULT_PREFLIGHT_MARKDOWN_OUT if args.phase == "preflight" else DEFAULT_POSTFLIGHT_MARKDOWN_OUT
)
return args
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
data = collect_state(args)
write_json(args.out, data)
write_markdown(args.markdown_out, data)
print(
json.dumps(
{
"out": str(args.out),
"markdown_out": str(args.markdown_out),
"phase": args.phase,
"status": data["status"],
"mutates_db": False,
},
indent=2,
sort_keys=True,
)
)
return 0 if data["status"] == "pass" else 2
if __name__ == "__main__":
raise SystemExit(main())