#!/usr/bin/env python3 """Restore a VPS canonical snapshot into one disposable private Cloud SQL database. The helper is intentionally narrower than a production cutover tool. It can create only a previously absent ``teleo_clone_*`` database, verifies full manifest parity, and deletes the clone automatically whenever restore or proof fails. A successful clone is retained only for a bounded no-send replay and is removed with the explicit ``cleanup`` operation afterward. """ from __future__ import annotations import argparse import hashlib import ipaddress import json import os import re import shlex import signal import stat import subprocess import time import urllib.error import urllib.request import uuid from datetime import UTC, datetime from pathlib import Path from typing import Any try: from .verify_postgres_parity_manifest import compare_manifests, load_manifest except ImportError: # pragma: no cover - direct script execution on the GCP VM from verify_postgres_parity_manifest import compare_manifests, load_manifest DEFAULT_HOST = "10.61.0.3" DEFAULT_PROJECT = "teleo-501523" DEFAULT_INSTANCE = "teleo-pgvector-standby" DEFAULT_PRIVATE_NETWORK = "projects/teleo-501523/global/networks/teleo-staging-net" DEFAULT_COMPUTE_INSTANCE = "teleo-prod-1" DEFAULT_COMPUTE_ZONE = "europe-west6-a" DEFAULT_SOURCE_SSH_TARGET = "root@77.42.65.182" DEFAULT_SOURCE_CONTAINER = "teleo-pg" DEFAULT_SOURCE_DATABASE = "teleo" DEFAULT_SOURCE_SERVICE = "leoclean-gateway.service" DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service" DEFAULT_RUN_ROOT = Path("/var/lib/teleo-gcp-restore-runs") DEFAULT_MANIFEST_SQL = Path(__file__).with_name("postgres_parity_manifest.sql") REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a" SOURCE_SNAPSHOT_RECEIPT_SCHEMA = "livingip.sourceSnapshotReceipt.v2" TARGET_DATABASE_RE = re.compile(r"teleo_clone_[a-z0-9][a-z0-9_]{0,50}\Z") RUN_ID_RE = re.compile(r"gcp-restore-[a-z0-9][a-z0-9-]{7,47}\Z") SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") LOCALE_RE = re.compile(r"[A-Za-z0-9_.@-]{1,64}\Z") CLOUDSQL_INSTANCE_RE = re.compile(r"[a-z][a-z0-9-]{0,97}[a-z0-9]\Z") AUTHORIZATION_REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:/@+-]{7,255}\Z") PRIVATE_NETWORK_RE = re.compile(r"projects/[a-z][a-z0-9-]{4,28}[a-z0-9]/global/networks/[a-z][a-z0-9-]{0,61}[a-z0-9]\Z") GCE_INSTANCE_RE = re.compile(r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?\Z") GCE_ZONE_RE = re.compile(r"[a-z]+-[a-z]+[0-9]-[a-z]\Z") SAFE_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z") SAFE_SSH_TARGET_RE = re.compile(r"(?:[A-Za-z0-9][A-Za-z0-9._-]{0,63}@)?[A-Za-z0-9][A-Za-z0-9.-]{0,252}\Z") PG_RESTORE_VERSION_RE = re.compile(r"pg_restore \(PostgreSQL\) (?P[0-9]+)(?:\.[0-9]+)*") ROLLBACK_DATABASE = "teleo_canonical_pre_20260712t1905z" GCE_METADATA_ROOT = "http://metadata.google.internal/computeMetadata/v1" class RestoreError(RuntimeError): """Raised when a guarded restore or cleanup invariant fails.""" class TerminationRequested(RestoreError): """Raised after SIGINT/SIGTERM so clone cleanup still runs.""" def utc_now() -> str: return datetime.now(UTC).isoformat() def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def canonical_sha256(value: Any) -> str: payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) return hashlib.sha256(payload.encode()).hexdigest() def validate_regular_file(path: Path, label: str) -> Path: try: mode = path.lstat().st_mode except OSError as exc: raise RestoreError(f"{label} is unavailable: {exc}") from exc if not stat.S_ISREG(mode) or path.is_symlink(): raise RestoreError(f"{label} must be a regular non-symlink file") return path.resolve() def validate_custom_dump(path: Path, expected_sha256: str) -> dict[str, Any]: path = validate_regular_file(path, "custom dump") if path.stat().st_size <= 5: raise RestoreError("custom dump is empty or truncated") with path.open("rb") as handle: if handle.read(5) != b"PGDMP": raise RestoreError("custom dump is missing the PGDMP signature") actual_sha256 = sha256_file(path) if actual_sha256 != expected_sha256: raise RestoreError("custom dump SHA-256 does not match --expected-dump-sha256") return {"path": str(path), "bytes": path.stat().st_size, "sha256": actual_sha256} def validate_source_snapshot_receipt( path: Path, *, expected_receipt_sha256: str, expected_authorization_ref: str, expected_source: dict[str, str], dump: dict[str, Any], source_manifest_path: Path, manifest_sql_path: Path, source_manifest: dict[str, Any], ) -> dict[str, Any]: path = validate_regular_file(path, "source snapshot receipt") receipt_sha256 = sha256_file(path) if receipt_sha256 != expected_receipt_sha256: raise RestoreError("source snapshot receipt SHA-256 does not match --expected-source-receipt-sha256") try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise RestoreError("source snapshot receipt is invalid JSON") from exc if not isinstance(payload, dict): raise RestoreError("source snapshot receipt must be a JSON object") snapshot = payload.get("snapshot") or {} manifest = payload.get("manifest") or {} source = payload.get("source") or {} source_context = payload.get("source_context") or {} source_service = payload.get("source_service") or {} binding = payload.get("provenance_binding") or {} binding_payload = binding.get("payload") or {} source_manifest_sha256 = sha256_file(source_manifest_path) manifest_sql_sha256 = sha256_file(manifest_sql_path) expected_table_count = len(source_manifest["tables"]) expected_total_rows = sum(int(row["row_count"]) for row in source_manifest["tables"].values()) expected_binding_payload = { "receipt_schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA, "run_id": payload.get("run_id"), "capture_authorization_ref": payload.get("capture_authorization_ref"), "source": source, "exported_snapshot_id": snapshot.get("exported_snapshot_id"), "txid_snapshot": snapshot.get("txid_snapshot"), "wal_lsn": snapshot.get("wal_lsn"), "source_system_identifier": snapshot.get("system_identifier"), "dump_sha256": dump["sha256"], "manifest_sql_sha256": manifest_sql_sha256, "source_manifest_sha256": source_manifest_sha256, "source_context_sha256": source_context.get("sha256"), } checks = { "artifact": payload.get("artifact") == "vps_canonical_postgres_exported_snapshot", "schema": payload.get("schema") == SOURCE_SNAPSHOT_RECEIPT_SCHEMA, "receipt_version": payload.get("receipt_version") == 2, "status": payload.get("status") == "pass", "snapshot_exported": payload.get("snapshot_exported") is True, "service_unchanged": payload.get("service_unchanged") is True, "source_service_healthy": source_service.get("unchanged") is True and service_state_is_healthy(source_service.get("before")) and source_service.get("before") == source_service.get("after"), "source_not_mutated": payload.get("production_db_mutated") is False, "telegram_not_sent": payload.get("telegram_send_attempted") is False, "run_id": bool(payload.get("run_id")), "capture_authorization_ref": payload.get("capture_authorization_ref") == expected_authorization_ref, "source_identity": source == expected_source, "source_database": source.get("database") == source_manifest["singleton"]["identity"].get("database"), "snapshot_identity": all( bool(snapshot.get(field)) for field in ("exported_snapshot_id", "txid_snapshot", "wal_lsn", "system_identifier", "captured_at_utc") ), "dump_sha256": (payload.get("dump") or {}).get("sha256") == dump["sha256"], "dump_bytes": (payload.get("dump") or {}).get("bytes") == dump["bytes"], "source_manifest_sha256": manifest.get("sha256") == source_manifest_sha256, "manifest_sql_sha256": manifest.get("manifest_sql_sha256") == manifest_sql_sha256, "source_context_sha256": bool(SHA256_RE.fullmatch(str(source_context.get("sha256") or ""))), "table_count": manifest.get("table_count") == expected_table_count, "total_rows": manifest.get("total_rows") == expected_total_rows, "binding_algorithm": binding.get("algorithm") == "sha256", "binding_payload": binding_payload == expected_binding_payload, "binding_sha256": binding.get("sha256") == canonical_sha256(expected_binding_payload), } failed = sorted(name for name, passed in checks.items() if not passed) if failed: raise RestoreError("source snapshot receipt failed validation: " + ", ".join(failed)) return { "path": str(path), "sha256": receipt_sha256, "schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA, "run_id": payload["run_id"], "capture_authorization_ref": payload["capture_authorization_ref"], "snapshot": snapshot, "provenance_binding_sha256": binding["sha256"], "manifest_sha256": source_manifest_sha256, "manifest_sql_sha256": manifest_sql_sha256, "dump_sha256": dump["sha256"], "table_count": expected_table_count, "total_rows": expected_total_rows, "checks": checks, } def _run( command: list[str], *, timeout: float, env: dict[str, str] | None = None, input_text: str | None = None, ) -> subprocess.CompletedProcess[str]: return subprocess.run( command, capture_output=True, check=False, env=env, input=input_text, text=True, timeout=timeout, ) def _bounded_error(completed: subprocess.CompletedProcess[str], secrets: tuple[str, ...] = ()) -> str: message = (completed.stderr or completed.stdout or "no command output").strip() for secret in secrets: if secret: message = message.replace(secret, "[REDACTED]") return message[-2000:] def _require_success( completed: subprocess.CompletedProcess[str], action: str, *, secrets: tuple[str, ...] = (), ) -> str: if completed.returncode != 0: raise RestoreError(f"{action} failed (exit {completed.returncode}): {_bounded_error(completed, secrets)}") return completed.stdout def service_state(service: str, *, timeout: float) -> dict[str, Any]: completed = _run( [ "systemctl", "show", service, "-p", "ActiveState", "-p", "SubState", "-p", "MainPID", "-p", "NRestarts", "-p", "ExecMainStartTimestamp", "--no-pager", ], timeout=timeout, ) raw = _require_success(completed, "live GCP service readback") result = {} for line in raw.splitlines(): key, separator, value = line.partition("=") if separator: result[key] = value required = {"ActiveState", "SubState", "MainPID", "NRestarts", "ExecMainStartTimestamp"} if required - set(result): raise RestoreError("live GCP service readback omitted required fields") return result def validate_service_healthy(state: dict[str, Any], label: str) -> None: if not service_state_is_healthy(state): raise RestoreError(f"{label} service is not active/running with a positive MainPID") def service_state_is_healthy(state: Any) -> bool: if not isinstance(state, dict): return False try: main_pid = int(state.get("MainPID") or 0) restarts = int(state.get("NRestarts") or 0) except (TypeError, ValueError): return False return ( state.get("ActiveState") == "active" and state.get("SubState") == "running" and main_pid > 0 and restarts >= 0 ) def cloudsql_control_plane_state( project: str, instance: str, expected_host: str, expected_private_network: str, *, timeout: float, ) -> dict[str, Any]: completed = _run( [ "gcloud", "sql", "instances", "describe", instance, f"--project={project}", "--format=json", "--quiet", ], timeout=timeout, ) raw = _require_success(completed, "Cloud SQL control-plane readback") try: payload = json.loads(raw) except json.JSONDecodeError as exc: raise RestoreError("Cloud SQL control-plane readback returned invalid JSON") from exc settings = payload.get("settings") or {} ip_configuration = settings.get("ipConfiguration") or {} addresses = payload.get("ipAddresses") or [] private_addresses = sorted( str(row.get("ipAddress")) for row in addresses if row.get("type") == "PRIVATE" and row.get("ipAddress") ) private_network = str(ip_configuration.get("privateNetwork") or "") non_private_addresses = sorted( str(row.get("ipAddress")) for row in addresses if row.get("type") != "PRIVATE" and row.get("ipAddress") ) checks = { "instance": payload.get("name") == instance, "state": payload.get("state") == "RUNNABLE", "postgres_16": str(payload.get("databaseVersion") or "").startswith("POSTGRES_16"), "public_ip_disabled": ip_configuration.get("ipv4Enabled") is False, "private_network": private_network == expected_private_network, "expected_private_host": expected_host in private_addresses, "no_non_private_address": not non_private_addresses, } failed = sorted(name for name, passed in checks.items() if not passed) if failed: raise RestoreError("Cloud SQL control-plane preflight failed: " + ", ".join(failed)) return { "project": project, "instance": instance, "database_version": payload.get("databaseVersion"), "region": payload.get("region"), "state": payload.get("state"), "tier": settings.get("tier"), "private_network": private_network, "expected_private_network": expected_private_network, "private_addresses": private_addresses, "expected_private_host": expected_host, "public_ip_disabled": True, "checks": checks, } def gce_metadata_value(path: str, *, timeout: float) -> str: request = urllib.request.Request( f"{GCE_METADATA_ROOT}/{path}", headers={"Metadata-Flavor": "Google"}, method="GET", ) opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) try: with opener.open(request, timeout=timeout) as response: if response.headers.get("Metadata-Flavor") != "Google": raise RestoreError("GCE metadata response is missing Metadata-Flavor: Google") value = response.read(4096).decode("utf-8", errors="strict").strip() except (OSError, UnicodeError, urllib.error.URLError) as exc: raise RestoreError(f"GCE metadata readback failed for {path}") from exc if not value or any(character in value for character in "\r\n\x00"): raise RestoreError(f"GCE metadata readback is invalid for {path}") return value def gce_compute_identity( project: str, expected_instance: str, expected_zone: str, expected_private_network: str, *, timeout: float, ) -> dict[str, Any]: metadata = { "project": gce_metadata_value("project/project-id", timeout=timeout), "project_number": gce_metadata_value("project/numeric-project-id", timeout=timeout), "instance": gce_metadata_value("instance/name", timeout=timeout), "instance_id": gce_metadata_value("instance/id", timeout=timeout), "zone_resource": gce_metadata_value("instance/zone", timeout=timeout), "network_resource": gce_metadata_value("instance/network-interfaces/0/network", timeout=timeout), "private_ip": gce_metadata_value("instance/network-interfaces/0/ip", timeout=timeout), "service_account": gce_metadata_value("instance/service-accounts/default/email", timeout=timeout), } zone = metadata["zone_resource"].rsplit("/", 1)[-1] network_name = expected_private_network.rsplit("/", 1)[-1] expected_network_resource = f"projects/{metadata['project_number']}/networks/{network_name}" try: private_ip = ipaddress.ip_address(metadata["private_ip"]) except ValueError as exc: raise RestoreError("GCE metadata private IP is invalid") from exc checks = { "project": metadata["project"] == project, "project_number": metadata["project_number"].isdigit(), "expected_network_project": expected_private_network.startswith(f"projects/{project}/global/networks/"), "instance": metadata["instance"] == expected_instance, "instance_id": metadata["instance_id"].isdigit(), "zone": zone == expected_zone, "network": metadata["network_resource"] == expected_network_resource, "private_ip": any( private_ip in network for network in ( ipaddress.ip_network("10.0.0.0/8"), ipaddress.ip_network("172.16.0.0/12"), ipaddress.ip_network("192.168.0.0/16"), ) ), "service_account": metadata["service_account"].endswith(".iam.gserviceaccount.com"), } failed = sorted(name for name, passed in checks.items() if not passed) if failed: raise RestoreError("GCE compute identity preflight failed: " + ", ".join(failed)) return { **metadata, "zone": zone, "expected_zone": expected_zone, "expected_private_network": expected_private_network, "expected_network_resource": expected_network_resource, "checks": checks, } def resolve_password(project: str, secret: str, *, timeout: float) -> str: completed = _run( [ "gcloud", "secrets", "versions", "access", "latest", f"--secret={secret}", f"--project={project}", "--quiet", ], timeout=timeout, ) password = _require_success(completed, "Cloud SQL credential resolution").strip() if not password: raise RestoreError("Cloud SQL credential resolved to an empty value") return password def validate_pg_restore_version( pg_restore_bin: str, source_manifest: dict[str, Any], *, timeout: float, ) -> dict[str, Any]: completed = _run([pg_restore_bin, "--version"], timeout=timeout) raw = _require_success(completed, "pg_restore version preflight").strip() match = PG_RESTORE_VERSION_RE.search(raw) if match is None: raise RestoreError("pg_restore version preflight returned an unrecognized value") client_major = int(match.group("major")) source_version_num = int(source_manifest["singleton"]["identity"]["server_version_num"]) source_major = source_version_num // 10000 if client_major < source_major: raise RestoreError(f"pg_restore major {client_major} is older than source PostgreSQL major {source_major}") return { "pg_restore_bin": pg_restore_bin, "pg_restore_version": raw, "pg_restore_major": client_major, "source_postgres_major": source_major, "compatible": True, } def postgres_env(password: str, *, read_only: bool) -> dict[str, str]: env = {**os.environ, "PGPASSWORD": password} options = ["-c statement_timeout=120000"] if read_only: options.append("-c default_transaction_read_only=on") env["PGOPTIONS"] = " ".join(options) return env def psql_command(args: argparse.Namespace, database: str) -> list[str]: return [ args.psql_bin, f"host={args.host} port=5432 dbname={database} user=postgres sslmode=require connect_timeout=8", "-X", "-Atq", "-v", "ON_ERROR_STOP=1", ] def run_psql( args: argparse.Namespace, password: str, database: str, sql: str, *, read_only: bool, timeout: float, ) -> str: completed = _run( psql_command(args, database), timeout=timeout, env=postgres_env(password, read_only=read_only), input_text=sql, ) return _require_success(completed, "PostgreSQL command", secrets=(password,)) def database_exists(args: argparse.Namespace, password: str, database: str) -> bool: output = run_psql( args, password, "postgres", f"select count(*) from pg_database where datname = '{database}';\n", read_only=True, timeout=args.command_timeout, ).strip() if output not in {"0", "1"}: raise RestoreError("database existence readback was invalid") return output == "1" def create_database( args: argparse.Namespace, password: str, source_manifest: dict[str, Any], ) -> None: identity = source_manifest["singleton"]["identity"] encoding = str(identity.get("server_encoding") or "") collation = str(identity.get("database_collation") or "") ctype = str(identity.get("database_ctype") or "") if encoding != "UTF8": raise RestoreError(f"source encoding is not the reviewed UTF8 value: {encoding!r}") if not LOCALE_RE.fullmatch(collation) or not LOCALE_RE.fullmatch(ctype): raise RestoreError("source collation or ctype is not a safe locale identifier") sql = ( f'create database "{args.target_db}" with template template0 ' f"encoding 'UTF8' lc_collate '{collation}' lc_ctype '{ctype}';\n" ) run_psql( args, password, "postgres", sql, read_only=False, timeout=args.command_timeout, ) def restore_dump(args: argparse.Namespace, password: str) -> None: command = [ args.pg_restore_bin, "-d", f"host={args.host} port=5432 dbname={args.target_db} user=postgres sslmode=require connect_timeout=8", "--no-owner", "--no-acl", "--exit-on-error", str(args.dump.resolve()), ] completed = _run( command, timeout=args.restore_timeout, env=postgres_env(password, read_only=False), ) _require_success(completed, "canonical PostgreSQL restore", secrets=(password,)) def capture_target_manifest(args: argparse.Namespace, password: str) -> str: command = [*psql_command(args, args.target_db), "-f", str(args.manifest_sql.resolve())] completed = _run( command, timeout=args.manifest_timeout, env=postgres_env(password, read_only=True), ) return _require_success(completed, "target parity manifest capture", secrets=(password,)) def validate_private_identity( target_manifest: dict[str, Any], target_db: str, source_compute: str, ) -> dict[str, Any]: identity = target_manifest["singleton"]["identity"] if identity.get("database") != target_db: raise RestoreError("target manifest database does not match the generated clone") if identity.get("ssl") is not True: raise RestoreError("target manifest did not prove TLS") if identity.get("transaction_read_only") != "on": raise RestoreError("target manifest was not captured in a read-only transaction") try: address = ipaddress.ip_address(str(identity.get("server_address") or "")) except ValueError as exc: raise RestoreError("target manifest server address is invalid") from exc private = any( address in network for network in ( ipaddress.ip_network("10.0.0.0/8"), ipaddress.ip_network("172.16.0.0/12"), ipaddress.ip_network("192.168.0.0/16"), ) ) if not private: raise RestoreError("target manifest server address is not RFC1918-private") return { "target_database": target_db, "server_address": str(address), "server_port": identity.get("server_port"), "ssl": True, "ssl_version": identity.get("ssl_version"), "private_connectivity": True, "transaction_read_only": "on", "source_compute": source_compute, } def drop_clone(args: argparse.Namespace, password: str) -> dict[str, Any]: if not TARGET_DATABASE_RE.fullmatch(args.target_db): raise RestoreError("refusing to drop a database outside teleo_clone_*") existed_before = database_exists(args, password, args.target_db) if existed_before: sql = f""" select pg_terminate_backend(pid) from pg_stat_activity where datname = '{args.target_db}' and pid <> pg_backend_pid(); drop database "{args.target_db}"; """ run_psql( args, password, "postgres", sql, read_only=False, timeout=args.command_timeout, ) remaining = 1 if database_exists(args, password, args.target_db) else 0 if remaining: raise RestoreError("generated clone remained after cleanup") return {"existed_before": existed_before, "clone_database_remaining": remaining} def rollback_database_state(args: argparse.Namespace, password: str) -> dict[str, Any]: output = run_psql( args, password, "postgres", f""" select jsonb_build_object( 'exists', (select count(*) = 1 from pg_database where datname = '{ROLLBACK_DATABASE}'), 'datallowconn', (select datallowconn from pg_database where datname = '{ROLLBACK_DATABASE}'), 'connections', (select count(*) from pg_stat_activity where datname = '{ROLLBACK_DATABASE}') )::text; """, read_only=True, timeout=args.command_timeout, ) rows = [line for line in output.splitlines() if line.strip().startswith("{")] if len(rows) != 1: raise RestoreError("rollback database state readback was invalid") state = json.loads(rows[0]) if state != {"exists": True, "datallowconn": False, "connections": 0}: raise RestoreError("disabled rollback database invariant changed") return state def secure_run_dir(run_root: Path, run_id: str, *, create: bool) -> Path: if run_root.is_symlink(): raise RestoreError("run root must not be a symlink") root = run_root.resolve() if create: root.mkdir(parents=True, exist_ok=True, mode=0o700) os.chmod(root, 0o700) if not root.is_dir() or root.is_symlink(): raise RestoreError("run root is absent or unsafe") run_dir = root / run_id if create: try: run_dir.mkdir(mode=0o700) except FileExistsError as exc: raise RestoreError("run directory already exists") from exc if not run_dir.is_dir() or run_dir.is_symlink() or run_dir.resolve().parent != root: raise RestoreError("run directory escaped the fixed root or is unsafe") os.chmod(run_dir, 0o700) return run_dir def write_private(path: Path, content: str | bytes) -> None: encoded = content.encode() if isinstance(content, str) else content temporary = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp" descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: with os.fdopen(descriptor, "wb") as handle: handle.write(encoded) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) os.chmod(path, 0o600) finally: try: temporary.unlink() except FileNotFoundError: pass def run_restore(args: argparse.Namespace) -> dict[str, Any]: started = time.monotonic() run_dir = secure_run_dir(args.run_root, args.run_id, create=True) receipt: dict[str, Any] = { "artifact": "gcp_generated_postgres_snapshot_restore", "generated_at_utc": utc_now(), "run_id": args.run_id, "target_database": args.target_db, "execution_contract": { "project": args.project, "cloudsql_instance": args.cloudsql_instance, "host": args.host, "private_network": args.expected_private_network, "compute_instance": args.expected_compute_instance, "compute_zone": args.expected_compute_zone, "source": { "ssh_target": args.expected_source_ssh_target, "container": args.expected_source_container, "database": args.expected_source_database, "service": args.expected_source_service, }, "password_secret": args.password_secret, "service": args.service, }, "status": "running", "phase": "validation", "source": {}, "toolchain": {}, "target": {}, "parity": {}, "live_service": {}, "rollback_database": {}, "cleanup": {"required": True, "attempted": False, "clone_database_remaining": None}, "safety": { "live_database_named": False, "live_profile_modified": False, "live_service_restarted": False, "telegram_message_sent": False, "secret_persisted": False, }, "error": None, } password = "" creation_attempted = False try: if os.geteuid() != 0: raise RestoreError("restore must run as root on the GCP staging VM") args.dump = validate_regular_file(args.dump, "custom dump") args.source_manifest = validate_regular_file(args.source_manifest, "source manifest") args.manifest_sql = validate_regular_file(args.manifest_sql, "parity manifest SQL") if sha256_file(args.manifest_sql) != REVIEWED_MANIFEST_SQL_SHA256: raise RestoreError("parity manifest SQL does not match the reviewed SHA-256") receipt["source"]["dump"] = validate_custom_dump(args.dump, args.expected_dump_sha256) receipt["source"]["manifest"] = { "path": str(args.source_manifest), "bytes": args.source_manifest.stat().st_size, "sha256": sha256_file(args.source_manifest), } source_manifest = load_manifest(args.source_manifest) receipt["source"]["capture_receipt"] = validate_source_snapshot_receipt( args.source_receipt, expected_receipt_sha256=args.expected_source_receipt_sha256, expected_authorization_ref=args.expected_capture_authorization_ref, expected_source=receipt["execution_contract"]["source"], dump=receipt["source"]["dump"], source_manifest_path=args.source_manifest, manifest_sql_path=args.manifest_sql, source_manifest=source_manifest, ) receipt["toolchain"] = validate_pg_restore_version( args.pg_restore_bin, source_manifest, timeout=args.command_timeout, ) receipt["target"]["compute"] = gce_compute_identity( args.project, args.expected_compute_instance, args.expected_compute_zone, args.expected_private_network, timeout=args.command_timeout, ) receipt["live_service"]["before"] = service_state(args.service, timeout=args.command_timeout) validate_service_healthy(receipt["live_service"]["before"], "pre-restore") receipt["phase"] = "cloudsql_control_plane_preflight" receipt["target"]["cloudsql"] = cloudsql_control_plane_state( args.project, args.cloudsql_instance, args.host, args.expected_private_network, timeout=args.command_timeout, ) receipt["phase"] = "credential_resolution" password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout) if database_exists(args, password, args.target_db): raise RestoreError("target clone already exists; refusing to overwrite it") receipt["phase"] = "database_create" creation_attempted = True create_database(args, password, source_manifest) receipt["phase"] = "pg_restore" restore_started = time.monotonic() restore_dump(args, password) receipt["target"]["restore_seconds"] = round(time.monotonic() - restore_started, 6) receipt["phase"] = "target_manifest" target_manifest_text = capture_target_manifest(args, password) target_manifest_path = run_dir / "target-manifest.jsonl" write_private(target_manifest_path, target_manifest_text) target_manifest = load_manifest(target_manifest_path) receipt["target"].update( { "manifest_path": str(target_manifest_path), "manifest_bytes": target_manifest_path.stat().st_size, "manifest_sha256": sha256_file(target_manifest_path), "connectivity": validate_private_identity( target_manifest, args.target_db, receipt["target"]["compute"]["instance"], ), } ) connectivity = receipt["target"]["connectivity"] cloudsql = receipt["target"]["cloudsql"] compute = receipt["target"]["compute"] connectivity_proof = { "artifact": "gcp_private_postgres_connectivity", "schema": "livingip.gcpPrivatePostgresConnectivity.v2", "generated_at_utc": utc_now(), "status": "pass", "restore_run_id": args.run_id, "target_database": args.target_db, "project": cloudsql["project"], "cloudsql_instance": cloudsql["instance"], "private_network": cloudsql["private_network"], "source_compute": compute["instance"], "source_compute_instance_id": compute["instance_id"], "source_compute_zone": compute["zone"], "source_compute_private_ip": compute["private_ip"], "server_address": connectivity["server_address"], "server_port": connectivity["server_port"], "ssl": connectivity["ssl"], "ssl_version": connectivity["ssl_version"], "private_connectivity": connectivity["private_connectivity"], "public_ip_disabled": cloudsql["public_ip_disabled"], "cloudsql_control_plane_checks": cloudsql["checks"], "compute_identity_checks": compute["checks"], } connectivity_proof_path = run_dir / "gcp-private-connectivity.json" write_private(connectivity_proof_path, json.dumps(connectivity_proof, indent=2, sort_keys=True) + "\n") receipt["target"]["connectivity_proof"] = { "path": str(connectivity_proof_path), "sha256": sha256_file(connectivity_proof_path), "schema": connectivity_proof["schema"], } receipt["phase"] = "parity_compare" problems, details = compare_manifests( source_manifest, target_manifest, max_target_query_ms=args.max_target_query_ms, max_target_source_ratio=args.max_target_source_ratio, ) receipt["parity"] = { "status": "pass" if not problems else "fail", "problems": problems, "details": details, "verifier": "ops.verify_postgres_parity_manifest.compare_manifests", } if problems: raise RestoreError("canonical source/target parity comparison failed") receipt["phase"] = "service_and_rollback_readback" receipt["rollback_database"] = rollback_database_state(args, password) receipt["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout) validate_service_healthy(receipt["live_service"]["after"], "post-restore") receipt["live_service"]["unchanged"] = receipt["live_service"]["before"] == receipt["live_service"]["after"] if not receipt["live_service"]["unchanged"]: raise RestoreError("live GCP service state changed during disposable restore") receipt["status"] = "pass" receipt["phase"] = "retained_for_no_send_replay" receipt["cleanup"] = { "required": True, "attempted": False, "clone_database_remaining": 1, "exact_command": shlex.join( [ "sudo", "python3", "ops/restore_gcp_generated_postgres_snapshot.py", "cleanup", "--execute", "--target-db", args.target_db, "--run-id", args.run_id, "--host", args.host, "--project", args.project, "--cloudsql-instance", args.cloudsql_instance, "--expected-private-network", args.expected_private_network, "--expected-compute-instance", args.expected_compute_instance, "--expected-compute-zone", args.expected_compute_zone, "--expected-source-ssh-target", args.expected_source_ssh_target, "--expected-source-container", args.expected_source_container, "--expected-source-database", args.expected_source_database, "--expected-source-service", args.expected_source_service, "--password-secret", args.password_secret, "--service", args.service, ] ), } except ( OSError, ValueError, KeyError, TypeError, json.JSONDecodeError, subprocess.TimeoutExpired, RestoreError, KeyboardInterrupt, ) as exc: receipt["status"] = "fail" receipt["error"] = { "phase": receipt.get("phase"), "type": type(exc).__name__, "message": str(exc)[-2000:], } if creation_attempted and password: receipt["cleanup"]["attempted"] = True try: receipt["cleanup"].update(drop_clone(args, password)) except Exception as cleanup_exc: # preserve both failures in the receipt receipt["cleanup"]["error"] = str(cleanup_exc)[-2000:] elif password: try: receipt["cleanup"]["clone_database_remaining"] = ( 1 if database_exists(args, password, args.target_db) else 0 ) except Exception as cleanup_readback_exc: receipt["cleanup"]["readback_error"] = str(cleanup_readback_exc)[-2000:] finally: if "after" not in receipt["live_service"] and "before" in receipt["live_service"]: try: receipt["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout) receipt["live_service"]["unchanged"] = ( receipt["live_service"]["before"] == receipt["live_service"]["after"] ) except Exception as service_exc: receipt["live_service"]["after_error"] = str(service_exc)[-2000:] receipt["duration_seconds"] = round(time.monotonic() - started, 6) receipt["completed_at_utc"] = utc_now() write_private(run_dir / "restore-receipt.json", json.dumps(receipt, indent=2, sort_keys=True) + "\n") password = "" return receipt def run_cleanup(args: argparse.Namespace) -> dict[str, Any]: run_dir = secure_run_dir(args.run_root, args.run_id, create=False) restore_receipt_path = run_dir / "restore-receipt.json" restore_receipt_path = validate_regular_file(restore_receipt_path, "restore receipt") restore_receipt_sha256 = sha256_file(restore_receipt_path) restore_receipt = json.loads(restore_receipt_path.read_text(encoding="utf-8")) if restore_receipt.get("status") != "pass": raise RestoreError("cleanup requires a passing restore receipt") if restore_receipt.get("target_database") != args.target_db: raise RestoreError("cleanup target does not match the retained restore receipt") if restore_receipt.get("run_id") != args.run_id: raise RestoreError("cleanup run id does not match the retained restore receipt") expected_contract = { "project": args.project, "cloudsql_instance": args.cloudsql_instance, "host": args.host, "private_network": args.expected_private_network, "compute_instance": args.expected_compute_instance, "compute_zone": args.expected_compute_zone, "source": { "ssh_target": args.expected_source_ssh_target, "container": args.expected_source_container, "database": args.expected_source_database, "service": args.expected_source_service, }, "password_secret": args.password_secret, "service": args.service, } if restore_receipt.get("execution_contract") != expected_contract: raise RestoreError("cleanup execution contract does not match the retained restore receipt") restore_cloudsql = (restore_receipt.get("target") or {}).get("cloudsql") or {} restore_topology_checks = { "project": restore_cloudsql.get("project") == args.project, "instance": restore_cloudsql.get("instance") == args.cloudsql_instance, "host": restore_cloudsql.get("expected_private_host") == args.host, "private_network": restore_cloudsql.get("expected_private_network") == args.expected_private_network, "public_ip_disabled": restore_cloudsql.get("public_ip_disabled") is True, } failed_topology_checks = sorted(name for name, passed in restore_topology_checks.items() if not passed) if failed_topology_checks: raise RestoreError( "cleanup target topology does not match the retained restore receipt: " + ", ".join(failed_topology_checks) ) restore_compute = (restore_receipt.get("target") or {}).get("compute") or {} restore_compute_checks = { "project": restore_compute.get("project") == args.project, "instance": restore_compute.get("instance") == args.expected_compute_instance, "zone": restore_compute.get("zone") == args.expected_compute_zone, "private_network": restore_compute.get("expected_private_network") == args.expected_private_network, "checks": bool(restore_compute.get("checks")) and all(value is True for value in restore_compute["checks"].values()), } failed_compute_checks = sorted(name for name, passed in restore_compute_checks.items() if not passed) if failed_compute_checks: raise RestoreError( "cleanup compute identity does not match the retained restore receipt: " + ", ".join(failed_compute_checks) ) started = time.monotonic() payload: dict[str, Any] = { "artifact": "gcp_generated_postgres_snapshot_cleanup", "generated_at_utc": utc_now(), "run_id": args.run_id, "target_database": args.target_db, "restore_receipt": {"path": str(restore_receipt_path), "sha256": restore_receipt_sha256}, "execution_contract": expected_contract, "target": {}, "status": "running", "phase": "cloudsql_control_plane_preflight", "database": {"clone_database_remaining": None}, "rollback_database": {}, "live_service": {}, "safety": { "live_database_named": False, "live_profile_modified": False, "live_service_restarted": False, "telegram_message_sent": False, "secret_persisted": False, }, "error": None, } password = "" try: payload["target"]["cloudsql"] = cloudsql_control_plane_state( args.project, args.cloudsql_instance, args.host, args.expected_private_network, timeout=args.command_timeout, ) payload["target"]["compute"] = gce_compute_identity( args.project, args.expected_compute_instance, args.expected_compute_zone, args.expected_private_network, timeout=args.command_timeout, ) cleanup_compute_mismatches = sorted( field for field in ( "project", "project_number", "instance", "instance_id", "zone", "network_resource", "private_ip", "service_account", ) if payload["target"]["compute"].get(field) != restore_compute.get(field) ) if cleanup_compute_mismatches: raise RestoreError("cleanup GCE identity changed since restore: " + ", ".join(cleanup_compute_mismatches)) payload["phase"] = "service_readback_before" payload["live_service"]["before"] = service_state(args.service, timeout=args.command_timeout) validate_service_healthy(payload["live_service"]["before"], "pre-cleanup") payload["phase"] = "credential_resolution" password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout) payload["phase"] = "database_drop" payload["database"] = drop_clone(args, password) payload["phase"] = "rollback_readback" payload["rollback_database"] = rollback_database_state(args, password) payload["phase"] = "service_readback_after" payload["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout) validate_service_healthy(payload["live_service"]["after"], "post-cleanup") payload["live_service"]["unchanged"] = payload["live_service"]["before"] == payload["live_service"]["after"] payload["status"] = ( "pass" if payload["live_service"]["unchanged"] and payload["database"]["clone_database_remaining"] == 0 else "fail" ) payload["phase"] = "complete" except ( OSError, ValueError, KeyError, TypeError, json.JSONDecodeError, subprocess.TimeoutExpired, RestoreError, KeyboardInterrupt, ) as exc: payload["status"] = "fail" payload["error"] = { "phase": payload.get("phase"), "type": type(exc).__name__, "message": str(exc)[-2000:], } if password and payload["database"].get("clone_database_remaining") is None: try: payload["database"]["clone_database_remaining"] = ( 1 if database_exists(args, password, args.target_db) else 0 ) except Exception as readback_exc: payload["database"]["readback_error"] = str(readback_exc)[-2000:] if "before" in payload["live_service"] and "after" not in payload["live_service"]: try: payload["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout) payload["live_service"]["unchanged"] = ( payload["live_service"]["before"] == payload["live_service"]["after"] ) except Exception as service_exc: payload["live_service"]["after_error"] = str(service_exc)[-2000:] finally: password = "" payload["duration_seconds"] = round(time.monotonic() - started, 6) payload["completed_at_utc"] = utc_now() write_private(run_dir / "cleanup-receipt.json", json.dumps(payload, indent=2, sort_keys=True) + "\n") return payload def add_common_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--execute", action="store_true") parser.add_argument("--target-db", required=True) parser.add_argument("--run-id", required=True) parser.add_argument("--host", default=DEFAULT_HOST) parser.add_argument("--project", default=DEFAULT_PROJECT) parser.add_argument("--cloudsql-instance", default=DEFAULT_INSTANCE) parser.add_argument("--expected-private-network", default=DEFAULT_PRIVATE_NETWORK) parser.add_argument("--expected-compute-instance", default=DEFAULT_COMPUTE_INSTANCE) parser.add_argument("--expected-compute-zone", default=DEFAULT_COMPUTE_ZONE) parser.add_argument("--expected-source-ssh-target", default=DEFAULT_SOURCE_SSH_TARGET) parser.add_argument("--expected-source-container", default=DEFAULT_SOURCE_CONTAINER) parser.add_argument("--expected-source-database", default=DEFAULT_SOURCE_DATABASE) parser.add_argument("--expected-source-service", default=DEFAULT_SOURCE_SERVICE) parser.add_argument("--password-secret", default=DEFAULT_SECRET) parser.add_argument("--service", default=DEFAULT_SERVICE) parser.add_argument("--command-timeout", type=float, default=120.0) parser.add_argument("--psql-bin", default="psql") parser.set_defaults(run_root=DEFAULT_RUN_ROOT) def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="operation", required=True) restore = subparsers.add_parser("restore", help="restore and retain one proven disposable clone") add_common_args(restore) restore.add_argument("--dump", required=True, type=Path) restore.add_argument("--expected-dump-sha256", required=True) restore.add_argument("--source-manifest", required=True, type=Path) restore.add_argument("--source-receipt", required=True, type=Path) restore.add_argument("--expected-source-receipt-sha256", required=True) restore.add_argument("--expected-capture-authorization-ref", required=True) restore.add_argument("--manifest-sql", default=DEFAULT_MANIFEST_SQL, type=Path) restore.add_argument("--pg-restore-bin", default="pg_restore") restore.add_argument("--restore-timeout", type=float, default=900.0) restore.add_argument("--manifest-timeout", type=float, default=300.0) restore.add_argument("--max-target-query-ms", type=float, default=2000.0) restore.add_argument("--max-target-source-ratio", type=float, default=20.0) cleanup = subparsers.add_parser("cleanup", help="drop the exact clone named by a passing receipt") add_common_args(cleanup) args = parser.parse_args(argv) if not args.execute: parser.error("--execute is required for Cloud SQL database lifecycle operations") if not TARGET_DATABASE_RE.fullmatch(args.target_db): parser.error("--target-db must be a bounded lowercase teleo_clone_* identifier") if not RUN_ID_RE.fullmatch(args.run_id): parser.error("--run-id must match gcp-restore-<8-48 lowercase letters, digits, or hyphens>") if not CLOUDSQL_INSTANCE_RE.fullmatch(args.cloudsql_instance): parser.error("--cloudsql-instance must be a safe lowercase Cloud SQL instance name") if not PRIVATE_NETWORK_RE.fullmatch(args.expected_private_network): parser.error("--expected-private-network must be a full projects/.../global/networks/... resource") if not args.expected_private_network.startswith(f"projects/{args.project}/global/networks/"): parser.error("--expected-private-network project must match --project") if not GCE_INSTANCE_RE.fullmatch(args.expected_compute_instance): parser.error("--expected-compute-instance must be a safe lowercase GCE instance name") if not GCE_ZONE_RE.fullmatch(args.expected_compute_zone): parser.error("--expected-compute-zone must be a safe GCE zone") if not SAFE_SSH_TARGET_RE.fullmatch(args.expected_source_ssh_target): parser.error("--expected-source-ssh-target must be a host or user@host without options") for value, flag in ( (args.expected_source_container, "--expected-source-container"), (args.expected_source_database, "--expected-source-database"), (args.expected_source_service, "--expected-source-service"), ): if not SAFE_NAME_RE.fullmatch(value): parser.error(f"{flag} contains unsafe characters") try: host = ipaddress.ip_address(args.host) except ValueError: parser.error("--host must be a literal IP address") rfc1918_networks = ( ipaddress.ip_network("10.0.0.0/8"), ipaddress.ip_network("172.16.0.0/12"), ipaddress.ip_network("192.168.0.0/16"), ) if not any(host in network for network in rfc1918_networks): parser.error("--host must be an RFC1918-private address") if args.command_timeout <= 0: parser.error("--command-timeout must be positive") if args.operation == "restore": if not SHA256_RE.fullmatch(args.expected_dump_sha256): parser.error("--expected-dump-sha256 must be 64 lowercase hexadecimal characters") if not SHA256_RE.fullmatch(args.expected_source_receipt_sha256): parser.error("--expected-source-receipt-sha256 must be 64 lowercase hexadecimal characters") if not AUTHORIZATION_REF_RE.fullmatch(args.expected_capture_authorization_ref): parser.error("--expected-capture-authorization-ref must be 8-256 safe metadata characters") for value, flag in ( (args.restore_timeout, "--restore-timeout"), (args.manifest_timeout, "--manifest-timeout"), (args.max_target_query_ms, "--max-target-query-ms"), (args.max_target_source_ratio, "--max-target-source-ratio"), ): if value <= 0: parser.error(f"{flag} must be positive") return args def _termination_handler(signum: int, _frame: Any) -> None: raise TerminationRequested(f"received {signal.Signals(signum).name}") def main(argv: list[str] | None = None) -> int: args = parse_args(argv) previous_handlers = {} for watched_signal in (signal.SIGINT, signal.SIGTERM): previous_handlers[watched_signal] = signal.getsignal(watched_signal) signal.signal(watched_signal, _termination_handler) try: payload = run_restore(args) if args.operation == "restore" else run_cleanup(args) finally: for watched_signal, previous in previous_handlers.items(): signal.signal(watched_signal, previous) print(json.dumps(payload, indent=2, sort_keys=True)) return 0 if payload.get("status") == "pass" else 1 if __name__ == "__main__": raise SystemExit(main())