#!/usr/bin/env python3 """Restore a VPS canonical snapshot into one disposable private Cloud SQL database. The helper is intentionally narrower than a production cutover tool. It can create only a previously absent ``teleo_clone_*`` database, verifies full manifest parity, and deletes the clone automatically whenever restore or proof fails. A successful clone is retained only for a bounded no-send replay and is removed with the explicit ``cleanup`` operation afterward. """ from __future__ import annotations import argparse import hashlib import ipaddress import json import os import re import signal import stat import subprocess import time import uuid from datetime import UTC, datetime from pathlib import Path from typing import Any try: from .verify_postgres_parity_manifest import compare_manifests, load_manifest except ImportError: # pragma: no cover - direct script execution on the GCP VM from verify_postgres_parity_manifest import compare_manifests, load_manifest DEFAULT_HOST = "10.61.0.3" DEFAULT_PROJECT = "teleo-501523" DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service" DEFAULT_RUN_ROOT = Path("/var/lib/teleo-gcp-restore-runs") DEFAULT_MANIFEST_SQL = Path(__file__).with_name("postgres_parity_manifest.sql") REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a" TARGET_DATABASE_RE = re.compile(r"teleo_clone_[a-z0-9][a-z0-9_]{0,50}\Z") RUN_ID_RE = re.compile(r"gcp-restore-[a-z0-9][a-z0-9-]{7,47}\Z") SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") LOCALE_RE = re.compile(r"[A-Za-z0-9_.@-]{1,64}\Z") PG_RESTORE_VERSION_RE = re.compile(r"pg_restore \(PostgreSQL\) (?P[0-9]+)(?:\.[0-9]+)*") ROLLBACK_DATABASE = "teleo_canonical_pre_20260712t1905z" class RestoreError(RuntimeError): """Raised when a guarded restore or cleanup invariant fails.""" class TerminationRequested(RestoreError): """Raised after SIGINT/SIGTERM so clone cleanup still runs.""" def utc_now() -> str: return datetime.now(UTC).isoformat() def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def validate_regular_file(path: Path, label: str) -> Path: try: mode = path.lstat().st_mode except OSError as exc: raise RestoreError(f"{label} is unavailable: {exc}") from exc if not stat.S_ISREG(mode) or path.is_symlink(): raise RestoreError(f"{label} must be a regular non-symlink file") return path.resolve() def validate_custom_dump(path: Path, expected_sha256: str) -> dict[str, Any]: path = validate_regular_file(path, "custom dump") if path.stat().st_size <= 5: raise RestoreError("custom dump is empty or truncated") with path.open("rb") as handle: if handle.read(5) != b"PGDMP": raise RestoreError("custom dump is missing the PGDMP signature") actual_sha256 = sha256_file(path) if actual_sha256 != expected_sha256: raise RestoreError("custom dump SHA-256 does not match --expected-dump-sha256") return {"path": str(path), "bytes": path.stat().st_size, "sha256": actual_sha256} def _run( command: list[str], *, timeout: float, env: dict[str, str] | None = None, input_text: str | None = None, ) -> subprocess.CompletedProcess[str]: return subprocess.run( command, capture_output=True, check=False, env=env, input=input_text, text=True, timeout=timeout, ) def _bounded_error(completed: subprocess.CompletedProcess[str], secrets: tuple[str, ...] = ()) -> str: message = (completed.stderr or completed.stdout or "no command output").strip() for secret in secrets: if secret: message = message.replace(secret, "[REDACTED]") return message[-2000:] def _require_success( completed: subprocess.CompletedProcess[str], action: str, *, secrets: tuple[str, ...] = (), ) -> str: if completed.returncode != 0: raise RestoreError(f"{action} failed (exit {completed.returncode}): {_bounded_error(completed, secrets)}") return completed.stdout def service_state(service: str, *, timeout: float) -> dict[str, Any]: completed = _run( [ "systemctl", "show", service, "-p", "ActiveState", "-p", "SubState", "-p", "MainPID", "-p", "NRestarts", "-p", "ExecMainStartTimestamp", "--no-pager", ], timeout=timeout, ) raw = _require_success(completed, "live GCP service readback") result = {} for line in raw.splitlines(): key, separator, value = line.partition("=") if separator: result[key] = value required = {"ActiveState", "SubState", "MainPID", "NRestarts", "ExecMainStartTimestamp"} if required - set(result): raise RestoreError("live GCP service readback omitted required fields") return result def resolve_password(project: str, secret: str, *, timeout: float) -> str: completed = _run( [ "gcloud", "secrets", "versions", "access", "latest", f"--secret={secret}", f"--project={project}", "--quiet", ], timeout=timeout, ) password = _require_success(completed, "Cloud SQL credential resolution").strip() if not password: raise RestoreError("Cloud SQL credential resolved to an empty value") return password def validate_pg_restore_version( pg_restore_bin: str, source_manifest: dict[str, Any], *, timeout: float, ) -> dict[str, Any]: completed = _run([pg_restore_bin, "--version"], timeout=timeout) raw = _require_success(completed, "pg_restore version preflight").strip() match = PG_RESTORE_VERSION_RE.search(raw) if match is None: raise RestoreError("pg_restore version preflight returned an unrecognized value") client_major = int(match.group("major")) source_version_num = int(source_manifest["singleton"]["identity"]["server_version_num"]) source_major = source_version_num // 10000 if client_major < source_major: raise RestoreError(f"pg_restore major {client_major} is older than source PostgreSQL major {source_major}") return { "pg_restore_bin": pg_restore_bin, "pg_restore_version": raw, "pg_restore_major": client_major, "source_postgres_major": source_major, "compatible": True, } def postgres_env(password: str, *, read_only: bool) -> dict[str, str]: env = {**os.environ, "PGPASSWORD": password} options = ["-c statement_timeout=120000"] if read_only: options.append("-c default_transaction_read_only=on") env["PGOPTIONS"] = " ".join(options) return env def psql_command(args: argparse.Namespace, database: str) -> list[str]: return [ args.psql_bin, f"host={args.host} port=5432 dbname={database} user=postgres sslmode=require connect_timeout=8", "-X", "-Atq", "-v", "ON_ERROR_STOP=1", ] def run_psql( args: argparse.Namespace, password: str, database: str, sql: str, *, read_only: bool, timeout: float, ) -> str: completed = _run( psql_command(args, database), timeout=timeout, env=postgres_env(password, read_only=read_only), input_text=sql, ) return _require_success(completed, "PostgreSQL command", secrets=(password,)) def database_exists(args: argparse.Namespace, password: str, database: str) -> bool: output = run_psql( args, password, "postgres", f"select count(*) from pg_database where datname = '{database}';\n", read_only=True, timeout=args.command_timeout, ).strip() if output not in {"0", "1"}: raise RestoreError("database existence readback was invalid") return output == "1" def create_database( args: argparse.Namespace, password: str, source_manifest: dict[str, Any], ) -> None: identity = source_manifest["singleton"]["identity"] encoding = str(identity.get("server_encoding") or "") collation = str(identity.get("database_collation") or "") ctype = str(identity.get("database_ctype") or "") if encoding != "UTF8": raise RestoreError(f"source encoding is not the reviewed UTF8 value: {encoding!r}") if not LOCALE_RE.fullmatch(collation) or not LOCALE_RE.fullmatch(ctype): raise RestoreError("source collation or ctype is not a safe locale identifier") sql = ( f'create database "{args.target_db}" with template template0 ' f"encoding 'UTF8' lc_collate '{collation}' lc_ctype '{ctype}';\n" ) run_psql( args, password, "postgres", sql, read_only=False, timeout=args.command_timeout, ) def restore_dump(args: argparse.Namespace, password: str) -> None: command = [ args.pg_restore_bin, "-d", f"host={args.host} port=5432 dbname={args.target_db} user=postgres sslmode=require connect_timeout=8", "--no-owner", "--no-acl", "--exit-on-error", str(args.dump.resolve()), ] completed = _run( command, timeout=args.restore_timeout, env=postgres_env(password, read_only=False), ) _require_success(completed, "canonical PostgreSQL restore", secrets=(password,)) def capture_target_manifest(args: argparse.Namespace, password: str) -> str: command = [*psql_command(args, args.target_db), "-f", str(args.manifest_sql.resolve())] completed = _run( command, timeout=args.manifest_timeout, env=postgres_env(password, read_only=True), ) return _require_success(completed, "target parity manifest capture", secrets=(password,)) def validate_private_identity(target_manifest: dict[str, Any], target_db: str) -> dict[str, Any]: identity = target_manifest["singleton"]["identity"] if identity.get("database") != target_db: raise RestoreError("target manifest database does not match the generated clone") if identity.get("ssl") is not True: raise RestoreError("target manifest did not prove TLS") if identity.get("transaction_read_only") != "on": raise RestoreError("target manifest was not captured in a read-only transaction") try: address = ipaddress.ip_address(str(identity.get("server_address") or "")) except ValueError as exc: raise RestoreError("target manifest server address is invalid") from exc private = any( address in network for network in ( ipaddress.ip_network("10.0.0.0/8"), ipaddress.ip_network("172.16.0.0/12"), ipaddress.ip_network("192.168.0.0/16"), ) ) if not private: raise RestoreError("target manifest server address is not RFC1918-private") return { "target_database": target_db, "server_address": str(address), "server_port": identity.get("server_port"), "ssl": True, "ssl_version": identity.get("ssl_version"), "private_connectivity": True, "transaction_read_only": "on", "source_compute": "teleo-prod-1", } def drop_clone(args: argparse.Namespace, password: str) -> dict[str, Any]: if not TARGET_DATABASE_RE.fullmatch(args.target_db): raise RestoreError("refusing to drop a database outside teleo_clone_*") existed_before = database_exists(args, password, args.target_db) if existed_before: sql = f""" select pg_terminate_backend(pid) from pg_stat_activity where datname = '{args.target_db}' and pid <> pg_backend_pid(); drop database "{args.target_db}"; """ run_psql( args, password, "postgres", sql, read_only=False, timeout=args.command_timeout, ) remaining = 1 if database_exists(args, password, args.target_db) else 0 if remaining: raise RestoreError("generated clone remained after cleanup") return {"existed_before": existed_before, "clone_database_remaining": remaining} def rollback_database_state(args: argparse.Namespace, password: str) -> dict[str, Any]: output = run_psql( args, password, "postgres", f""" select jsonb_build_object( 'exists', (select count(*) = 1 from pg_database where datname = '{ROLLBACK_DATABASE}'), 'datallowconn', (select datallowconn from pg_database where datname = '{ROLLBACK_DATABASE}'), 'connections', (select count(*) from pg_stat_activity where datname = '{ROLLBACK_DATABASE}') )::text; """, read_only=True, timeout=args.command_timeout, ) rows = [line for line in output.splitlines() if line.strip().startswith("{")] if len(rows) != 1: raise RestoreError("rollback database state readback was invalid") state = json.loads(rows[0]) if state != {"exists": True, "datallowconn": False, "connections": 0}: raise RestoreError("disabled rollback database invariant changed") return state def secure_run_dir(run_root: Path, run_id: str, *, create: bool) -> Path: if run_root.is_symlink(): raise RestoreError("run root must not be a symlink") root = run_root.resolve() if create: root.mkdir(parents=True, exist_ok=True, mode=0o700) os.chmod(root, 0o700) if not root.is_dir() or root.is_symlink(): raise RestoreError("run root is absent or unsafe") run_dir = root / run_id if create: try: run_dir.mkdir(mode=0o700) except FileExistsError as exc: raise RestoreError("run directory already exists") from exc if not run_dir.is_dir() or run_dir.is_symlink() or run_dir.resolve().parent != root: raise RestoreError("run directory escaped the fixed root or is unsafe") os.chmod(run_dir, 0o700) return run_dir def write_private(path: Path, content: str | bytes) -> None: encoded = content.encode() if isinstance(content, str) else content temporary = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp" descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: with os.fdopen(descriptor, "wb") as handle: handle.write(encoded) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) os.chmod(path, 0o600) finally: try: temporary.unlink() except FileNotFoundError: pass def run_restore(args: argparse.Namespace) -> dict[str, Any]: started = time.monotonic() run_dir = secure_run_dir(args.run_root, args.run_id, create=True) receipt: dict[str, Any] = { "artifact": "gcp_generated_postgres_snapshot_restore", "generated_at_utc": utc_now(), "run_id": args.run_id, "target_database": args.target_db, "status": "running", "phase": "validation", "source": {}, "toolchain": {}, "target": {}, "parity": {}, "live_service": {}, "rollback_database": {}, "cleanup": {"required": True, "attempted": False, "clone_database_remaining": None}, "safety": { "live_database_named": False, "live_profile_modified": False, "live_service_restarted": False, "telegram_message_sent": False, "secret_persisted": False, }, "error": None, } password = "" creation_attempted = False try: if os.geteuid() != 0: raise RestoreError("restore must run as root on the GCP staging VM") args.dump = validate_regular_file(args.dump, "custom dump") args.source_manifest = validate_regular_file(args.source_manifest, "source manifest") args.manifest_sql = validate_regular_file(args.manifest_sql, "parity manifest SQL") if sha256_file(args.manifest_sql) != REVIEWED_MANIFEST_SQL_SHA256: raise RestoreError("parity manifest SQL does not match the reviewed SHA-256") receipt["source"]["dump"] = validate_custom_dump(args.dump, args.expected_dump_sha256) receipt["source"]["manifest"] = { "path": str(args.source_manifest), "bytes": args.source_manifest.stat().st_size, "sha256": sha256_file(args.source_manifest), } source_manifest = load_manifest(args.source_manifest) receipt["toolchain"] = validate_pg_restore_version( args.pg_restore_bin, source_manifest, timeout=args.command_timeout, ) receipt["live_service"]["before"] = service_state(args.service, timeout=args.command_timeout) receipt["phase"] = "credential_resolution" password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout) if database_exists(args, password, args.target_db): raise RestoreError("target clone already exists; refusing to overwrite it") receipt["phase"] = "database_create" creation_attempted = True create_database(args, password, source_manifest) receipt["phase"] = "pg_restore" restore_started = time.monotonic() restore_dump(args, password) receipt["target"]["restore_seconds"] = round(time.monotonic() - restore_started, 6) receipt["phase"] = "target_manifest" target_manifest_text = capture_target_manifest(args, password) target_manifest_path = run_dir / "target-manifest.jsonl" write_private(target_manifest_path, target_manifest_text) target_manifest = load_manifest(target_manifest_path) receipt["target"].update( { "manifest_path": str(target_manifest_path), "manifest_bytes": target_manifest_path.stat().st_size, "manifest_sha256": sha256_file(target_manifest_path), "connectivity": validate_private_identity(target_manifest, args.target_db), } ) receipt["phase"] = "parity_compare" problems, details = compare_manifests( source_manifest, target_manifest, max_target_query_ms=args.max_target_query_ms, max_target_source_ratio=args.max_target_source_ratio, ) receipt["parity"] = { "status": "pass" if not problems else "fail", "problems": problems, "details": details, "verifier": "ops.verify_postgres_parity_manifest.compare_manifests", } if problems: raise RestoreError("canonical source/target parity comparison failed") receipt["phase"] = "service_and_rollback_readback" receipt["rollback_database"] = rollback_database_state(args, password) receipt["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout) receipt["live_service"]["unchanged"] = receipt["live_service"]["before"] == receipt["live_service"]["after"] if not receipt["live_service"]["unchanged"]: raise RestoreError("live GCP service state changed during disposable restore") receipt["status"] = "pass" receipt["phase"] = "retained_for_no_send_replay" receipt["cleanup"] = { "required": True, "attempted": False, "clone_database_remaining": 1, "exact_command": ( f"sudo python3 ops/restore_gcp_generated_postgres_snapshot.py cleanup --execute " f"--target-db {args.target_db} --run-id {args.run_id}" ), } except ( OSError, ValueError, KeyError, TypeError, json.JSONDecodeError, subprocess.TimeoutExpired, RestoreError, KeyboardInterrupt, ) as exc: receipt["status"] = "fail" receipt["error"] = { "phase": receipt.get("phase"), "type": type(exc).__name__, "message": str(exc)[-2000:], } if creation_attempted and password: receipt["cleanup"]["attempted"] = True try: receipt["cleanup"].update(drop_clone(args, password)) except Exception as cleanup_exc: # preserve both failures in the receipt receipt["cleanup"]["error"] = str(cleanup_exc)[-2000:] elif password: try: receipt["cleanup"]["clone_database_remaining"] = ( 1 if database_exists(args, password, args.target_db) else 0 ) except Exception as cleanup_readback_exc: receipt["cleanup"]["readback_error"] = str(cleanup_readback_exc)[-2000:] finally: if "after" not in receipt["live_service"] and "before" in receipt["live_service"]: try: receipt["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout) receipt["live_service"]["unchanged"] = ( receipt["live_service"]["before"] == receipt["live_service"]["after"] ) except Exception as service_exc: receipt["live_service"]["after_error"] = str(service_exc)[-2000:] receipt["duration_seconds"] = round(time.monotonic() - started, 6) write_private(run_dir / "restore-receipt.json", json.dumps(receipt, indent=2, sort_keys=True) + "\n") password = "" return receipt def run_cleanup(args: argparse.Namespace) -> dict[str, Any]: run_dir = secure_run_dir(args.run_root, args.run_id, create=False) restore_receipt_path = run_dir / "restore-receipt.json" restore_receipt_path = validate_regular_file(restore_receipt_path, "restore receipt") restore_receipt = json.loads(restore_receipt_path.read_text(encoding="utf-8")) if restore_receipt.get("status") != "pass": raise RestoreError("cleanup requires a passing restore receipt") if restore_receipt.get("target_database") != args.target_db: raise RestoreError("cleanup target does not match the retained restore receipt") if restore_receipt.get("run_id") != args.run_id: raise RestoreError("cleanup run id does not match the retained restore receipt") before = service_state(args.service, timeout=args.command_timeout) password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout) try: dropped = drop_clone(args, password) rollback = rollback_database_state(args, password) finally: password = "" after = service_state(args.service, timeout=args.command_timeout) payload = { "artifact": "gcp_generated_postgres_snapshot_cleanup", "generated_at_utc": utc_now(), "run_id": args.run_id, "target_database": args.target_db, "status": "pass" if before == after and dropped["clone_database_remaining"] == 0 else "fail", "database": dropped, "rollback_database": rollback, "live_service": {"before": before, "after": after, "unchanged": before == after}, "safety": { "live_database_named": False, "live_profile_modified": False, "live_service_restarted": False, "telegram_message_sent": False, "secret_persisted": False, }, } write_private(run_dir / "cleanup-receipt.json", json.dumps(payload, indent=2, sort_keys=True) + "\n") return payload def add_common_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--execute", action="store_true") parser.add_argument("--target-db", required=True) parser.add_argument("--run-id", required=True) parser.add_argument("--host", default=DEFAULT_HOST) parser.add_argument("--project", default=DEFAULT_PROJECT) parser.add_argument("--password-secret", default=DEFAULT_SECRET) parser.add_argument("--service", default=DEFAULT_SERVICE) parser.add_argument("--command-timeout", type=float, default=120.0) parser.add_argument("--psql-bin", default="psql") parser.set_defaults(run_root=DEFAULT_RUN_ROOT) def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="operation", required=True) restore = subparsers.add_parser("restore", help="restore and retain one proven disposable clone") add_common_args(restore) restore.add_argument("--dump", required=True, type=Path) restore.add_argument("--expected-dump-sha256", required=True) restore.add_argument("--source-manifest", required=True, type=Path) restore.add_argument("--manifest-sql", default=DEFAULT_MANIFEST_SQL, type=Path) restore.add_argument("--pg-restore-bin", default="pg_restore") restore.add_argument("--restore-timeout", type=float, default=900.0) restore.add_argument("--manifest-timeout", type=float, default=300.0) restore.add_argument("--max-target-query-ms", type=float, default=2000.0) restore.add_argument("--max-target-source-ratio", type=float, default=20.0) cleanup = subparsers.add_parser("cleanup", help="drop the exact clone named by a passing receipt") add_common_args(cleanup) args = parser.parse_args(argv) if not args.execute: parser.error("--execute is required for Cloud SQL database lifecycle operations") if not TARGET_DATABASE_RE.fullmatch(args.target_db): parser.error("--target-db must be a bounded lowercase teleo_clone_* identifier") if not RUN_ID_RE.fullmatch(args.run_id): parser.error("--run-id must match gcp-restore-<8-48 lowercase letters, digits, or hyphens>") try: host = ipaddress.ip_address(args.host) except ValueError: parser.error("--host must be a literal IP address") rfc1918_networks = ( ipaddress.ip_network("10.0.0.0/8"), ipaddress.ip_network("172.16.0.0/12"), ipaddress.ip_network("192.168.0.0/16"), ) if not any(host in network for network in rfc1918_networks): parser.error("--host must be an RFC1918-private address") if args.command_timeout <= 0: parser.error("--command-timeout must be positive") if args.operation == "restore": if not SHA256_RE.fullmatch(args.expected_dump_sha256): parser.error("--expected-dump-sha256 must be 64 lowercase hexadecimal characters") for value, flag in ( (args.restore_timeout, "--restore-timeout"), (args.manifest_timeout, "--manifest-timeout"), (args.max_target_query_ms, "--max-target-query-ms"), (args.max_target_source_ratio, "--max-target-source-ratio"), ): if value <= 0: parser.error(f"{flag} must be positive") return args def _termination_handler(signum: int, _frame: Any) -> None: raise TerminationRequested(f"received {signal.Signals(signum).name}") def main(argv: list[str] | None = None) -> int: args = parse_args(argv) previous_handlers = {} for watched_signal in (signal.SIGINT, signal.SIGTERM): previous_handlers[watched_signal] = signal.getsignal(watched_signal) signal.signal(watched_signal, _termination_handler) try: payload = run_restore(args) if args.operation == "restore" else run_cleanup(args) finally: for watched_signal, previous in previous_handlers.items(): signal.signal(watched_signal, previous) print(json.dumps(payload, indent=2, sort_keys=True)) return 0 if payload.get("status") == "pass" else 1 if __name__ == "__main__": raise SystemExit(main())