1353 lines
58 KiB
Python
1353 lines
58 KiB
Python
#!/usr/bin/env python3
|
|
"""Restore a VPS canonical snapshot into one disposable private Cloud SQL database.
|
|
|
|
The helper is intentionally narrower than a production cutover tool. It can
|
|
create only a previously absent ``teleo_clone_*`` database, verifies full
|
|
manifest parity, and deletes the clone automatically whenever restore or proof
|
|
fails. A successful clone is retained only for a bounded no-send replay and is
|
|
removed with the explicit ``cleanup`` operation afterward.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import signal
|
|
import stat
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
try:
|
|
from .private_receipt_io import print_private_receipt_summary
|
|
from .verify_postgres_parity_manifest import REVIEWED_MANIFEST_SQL_SHA256, compare_manifests, load_manifest
|
|
except ImportError: # pragma: no cover - direct script execution on the GCP VM
|
|
from private_receipt_io import print_private_receipt_summary
|
|
from verify_postgres_parity_manifest import REVIEWED_MANIFEST_SQL_SHA256, compare_manifests, load_manifest
|
|
|
|
|
|
DEFAULT_HOST = "10.61.0.3"
|
|
DEFAULT_PROJECT = "teleo-501523"
|
|
DEFAULT_INSTANCE = "teleo-pgvector-standby"
|
|
DEFAULT_PRIVATE_NETWORK = "projects/teleo-501523/global/networks/teleo-staging-net"
|
|
DEFAULT_COMPUTE_INSTANCE = "teleo-prod-1"
|
|
DEFAULT_COMPUTE_ZONE = "europe-west6-a"
|
|
DEFAULT_RESTORE_SERVICE_ACCOUNT = "sa-teleo-restore-drill@teleo-501523.iam.gserviceaccount.com"
|
|
DEFAULT_SOURCE_SSH_TARGET = "root@77.42.65.182"
|
|
DEFAULT_SOURCE_CONTAINER = "teleo-pg"
|
|
DEFAULT_SOURCE_DATABASE = "teleo"
|
|
DEFAULT_SOURCE_SERVICE = "leoclean-gateway.service"
|
|
DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password"
|
|
DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service"
|
|
DEFAULT_RUN_ROOT = Path("/var/lib/teleo-gcp-restore-runs")
|
|
DEFAULT_MANIFEST_SQL = Path(__file__).with_name("postgres_parity_manifest.sql")
|
|
SOURCE_SNAPSHOT_RECEIPT_SCHEMA = "livingip.sourceSnapshotReceipt.v2"
|
|
TARGET_DATABASE_RE = re.compile(r"teleo_clone_[a-z0-9][a-z0-9_]{0,50}\Z")
|
|
RUN_ID_RE = re.compile(r"gcp-restore-[a-z0-9][a-z0-9-]{7,47}\Z")
|
|
SHA256_RE = re.compile(r"[0-9a-f]{64}\Z")
|
|
LOCALE_RE = re.compile(r"[A-Za-z0-9_.@-]{1,64}\Z")
|
|
CLOUDSQL_INSTANCE_RE = re.compile(r"[a-z][a-z0-9-]{0,97}[a-z0-9]\Z")
|
|
AUTHORIZATION_REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:/@+-]{7,255}\Z")
|
|
PRIVATE_NETWORK_RE = re.compile(r"projects/[a-z][a-z0-9-]{4,28}[a-z0-9]/global/networks/[a-z][a-z0-9-]{0,61}[a-z0-9]\Z")
|
|
GCE_INSTANCE_RE = re.compile(r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?\Z")
|
|
GCE_ZONE_RE = re.compile(r"[a-z]+-[a-z]+[0-9]-[a-z]\Z")
|
|
SAFE_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
|
|
SAFE_SSH_TARGET_RE = re.compile(r"(?:[A-Za-z0-9][A-Za-z0-9._-]{0,63}@)?[A-Za-z0-9][A-Za-z0-9.-]{0,252}\Z")
|
|
SAFE_DNS_NAME_RE = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?\Z")
|
|
SERVICE_ACCOUNT_RE = re.compile(r"[a-z][a-z0-9-]{0,62}@[a-z][a-z0-9-]{4,28}[a-z0-9]\.iam\.gserviceaccount\.com\Z")
|
|
PG_RESTORE_VERSION_RE = re.compile(r"pg_restore \(PostgreSQL\) (?P<major>[0-9]+)(?:\.[0-9]+)*")
|
|
ROLLBACK_DATABASE = "teleo_canonical_pre_20260712t1905z"
|
|
GCE_METADATA_ROOT = "http://metadata.google.internal/computeMetadata/v1"
|
|
|
|
|
|
class RestoreError(RuntimeError):
|
|
"""Raised when a guarded restore or cleanup invariant fails."""
|
|
|
|
|
|
class TerminationRequested(RestoreError):
|
|
"""Raised after SIGINT/SIGTERM so clone cleanup still runs."""
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(UTC).isoformat()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def canonical_sha256(value: Any) -> str:
|
|
payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
|
return hashlib.sha256(payload.encode()).hexdigest()
|
|
|
|
|
|
def validate_regular_file(path: Path, label: str) -> Path:
|
|
try:
|
|
mode = path.lstat().st_mode
|
|
except OSError as exc:
|
|
raise RestoreError(f"{label} is unavailable: {exc}") from exc
|
|
if not stat.S_ISREG(mode) or path.is_symlink():
|
|
raise RestoreError(f"{label} must be a regular non-symlink file")
|
|
return path.resolve()
|
|
|
|
|
|
def validate_custom_dump(path: Path, expected_sha256: str) -> dict[str, Any]:
|
|
path = validate_regular_file(path, "custom dump")
|
|
if path.stat().st_size <= 5:
|
|
raise RestoreError("custom dump is empty or truncated")
|
|
with path.open("rb") as handle:
|
|
if handle.read(5) != b"PGDMP":
|
|
raise RestoreError("custom dump is missing the PGDMP signature")
|
|
actual_sha256 = sha256_file(path)
|
|
if actual_sha256 != expected_sha256:
|
|
raise RestoreError("custom dump SHA-256 does not match --expected-dump-sha256")
|
|
return {"path": str(path), "bytes": path.stat().st_size, "sha256": actual_sha256}
|
|
|
|
|
|
def validate_source_snapshot_receipt(
|
|
path: Path,
|
|
*,
|
|
expected_receipt_sha256: str,
|
|
expected_authorization_ref: str,
|
|
expected_source: dict[str, str],
|
|
dump: dict[str, Any],
|
|
source_manifest_path: Path,
|
|
manifest_sql_path: Path,
|
|
source_manifest: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
path = validate_regular_file(path, "source snapshot receipt")
|
|
receipt_sha256 = sha256_file(path)
|
|
if receipt_sha256 != expected_receipt_sha256:
|
|
raise RestoreError("source snapshot receipt SHA-256 does not match --expected-source-receipt-sha256")
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
except json.JSONDecodeError as exc:
|
|
raise RestoreError("source snapshot receipt is invalid JSON") from exc
|
|
if not isinstance(payload, dict):
|
|
raise RestoreError("source snapshot receipt must be a JSON object")
|
|
|
|
snapshot = payload.get("snapshot") or {}
|
|
manifest = payload.get("manifest") or {}
|
|
source = payload.get("source") or {}
|
|
source_context = payload.get("source_context") or {}
|
|
source_service = payload.get("source_service") or {}
|
|
binding = payload.get("provenance_binding") or {}
|
|
binding_payload = binding.get("payload") or {}
|
|
source_manifest_sha256 = sha256_file(source_manifest_path)
|
|
manifest_sql_sha256 = sha256_file(manifest_sql_path)
|
|
expected_table_count = len(source_manifest["tables"])
|
|
expected_total_rows = sum(int(row["row_count"]) for row in source_manifest["tables"].values())
|
|
expected_binding_payload = {
|
|
"receipt_schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA,
|
|
"run_id": payload.get("run_id"),
|
|
"capture_authorization_ref": payload.get("capture_authorization_ref"),
|
|
"source": source,
|
|
"exported_snapshot_id": snapshot.get("exported_snapshot_id"),
|
|
"txid_snapshot": snapshot.get("txid_snapshot"),
|
|
"wal_lsn": snapshot.get("wal_lsn"),
|
|
"source_system_identifier": snapshot.get("system_identifier"),
|
|
"dump_sha256": dump["sha256"],
|
|
"manifest_sql_sha256": manifest_sql_sha256,
|
|
"source_manifest_sha256": source_manifest_sha256,
|
|
"source_context_sha256": source_context.get("sha256"),
|
|
}
|
|
checks = {
|
|
"artifact": payload.get("artifact") == "vps_canonical_postgres_exported_snapshot",
|
|
"schema": payload.get("schema") == SOURCE_SNAPSHOT_RECEIPT_SCHEMA,
|
|
"receipt_version": payload.get("receipt_version") == 2,
|
|
"status": payload.get("status") == "pass",
|
|
"snapshot_exported": payload.get("snapshot_exported") is True,
|
|
"service_unchanged": payload.get("service_unchanged") is True,
|
|
"source_service_healthy": source_service.get("unchanged") is True
|
|
and service_state_is_healthy(source_service.get("before"))
|
|
and source_service.get("before") == source_service.get("after"),
|
|
"source_not_mutated": payload.get("production_db_mutated") is False,
|
|
"telegram_not_sent": payload.get("telegram_send_attempted") is False,
|
|
"run_id": bool(payload.get("run_id")),
|
|
"capture_authorization_ref": payload.get("capture_authorization_ref") == expected_authorization_ref,
|
|
"source_identity": source == expected_source,
|
|
"source_database": source.get("database") == source_manifest["singleton"]["identity"].get("database"),
|
|
"snapshot_identity": all(
|
|
bool(snapshot.get(field))
|
|
for field in ("exported_snapshot_id", "txid_snapshot", "wal_lsn", "system_identifier", "captured_at_utc")
|
|
),
|
|
"dump_sha256": (payload.get("dump") or {}).get("sha256") == dump["sha256"],
|
|
"dump_bytes": (payload.get("dump") or {}).get("bytes") == dump["bytes"],
|
|
"source_manifest_sha256": manifest.get("sha256") == source_manifest_sha256,
|
|
"manifest_sql_sha256": manifest.get("manifest_sql_sha256") == manifest_sql_sha256,
|
|
"source_context_sha256": bool(SHA256_RE.fullmatch(str(source_context.get("sha256") or ""))),
|
|
"table_count": manifest.get("table_count") == expected_table_count,
|
|
"total_rows": manifest.get("total_rows") == expected_total_rows,
|
|
"binding_algorithm": binding.get("algorithm") == "sha256",
|
|
"binding_payload": binding_payload == expected_binding_payload,
|
|
"binding_sha256": binding.get("sha256") == canonical_sha256(expected_binding_payload),
|
|
}
|
|
failed = sorted(name for name, passed in checks.items() if not passed)
|
|
if failed:
|
|
raise RestoreError("source snapshot receipt failed validation: " + ", ".join(failed))
|
|
return {
|
|
"path": str(path),
|
|
"sha256": receipt_sha256,
|
|
"schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA,
|
|
"run_id": payload["run_id"],
|
|
"capture_authorization_ref": payload["capture_authorization_ref"],
|
|
"snapshot": snapshot,
|
|
"provenance_binding_sha256": binding["sha256"],
|
|
"manifest_sha256": source_manifest_sha256,
|
|
"manifest_sql_sha256": manifest_sql_sha256,
|
|
"dump_sha256": dump["sha256"],
|
|
"table_count": expected_table_count,
|
|
"total_rows": expected_total_rows,
|
|
"checks": checks,
|
|
}
|
|
|
|
|
|
def _run(
|
|
command: list[str],
|
|
*,
|
|
timeout: float,
|
|
env: dict[str, str] | None = None,
|
|
input_text: str | None = None,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
command,
|
|
capture_output=True,
|
|
check=False,
|
|
env=env,
|
|
input=input_text,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
|
|
|
|
def _bounded_error(completed: subprocess.CompletedProcess[str], secrets: tuple[str, ...] = ()) -> str:
|
|
message = (completed.stderr or completed.stdout or "no command output").strip()
|
|
for secret in secrets:
|
|
if secret:
|
|
message = message.replace(secret, "[REDACTED]")
|
|
return message[-2000:]
|
|
|
|
|
|
def _require_success(
|
|
completed: subprocess.CompletedProcess[str],
|
|
action: str,
|
|
*,
|
|
secrets: tuple[str, ...] = (),
|
|
) -> str:
|
|
if completed.returncode != 0:
|
|
raise RestoreError(f"{action} failed (exit {completed.returncode}): {_bounded_error(completed, secrets)}")
|
|
return completed.stdout
|
|
|
|
|
|
def service_state(service: str, *, timeout: float) -> dict[str, Any]:
|
|
completed = _run(
|
|
[
|
|
"systemctl",
|
|
"show",
|
|
service,
|
|
"-p",
|
|
"ActiveState",
|
|
"-p",
|
|
"SubState",
|
|
"-p",
|
|
"MainPID",
|
|
"-p",
|
|
"NRestarts",
|
|
"-p",
|
|
"ExecMainStartTimestamp",
|
|
"--no-pager",
|
|
],
|
|
timeout=timeout,
|
|
)
|
|
raw = _require_success(completed, "live GCP service readback")
|
|
result = {}
|
|
for line in raw.splitlines():
|
|
key, separator, value = line.partition("=")
|
|
if separator:
|
|
result[key] = value
|
|
required = {"ActiveState", "SubState", "MainPID", "NRestarts", "ExecMainStartTimestamp"}
|
|
if required - set(result):
|
|
raise RestoreError("live GCP service readback omitted required fields")
|
|
return result
|
|
|
|
|
|
def validate_service_healthy(state: dict[str, Any], label: str) -> None:
|
|
if not service_state_is_healthy(state):
|
|
raise RestoreError(f"{label} service is not active/running with a positive MainPID")
|
|
|
|
|
|
def service_state_is_healthy(state: Any) -> bool:
|
|
if not isinstance(state, dict):
|
|
return False
|
|
try:
|
|
main_pid = int(state.get("MainPID") or 0)
|
|
restarts = int(state.get("NRestarts") or 0)
|
|
except (TypeError, ValueError):
|
|
return False
|
|
return (
|
|
state.get("ActiveState") == "active" and state.get("SubState") == "running" and main_pid > 0 and restarts >= 0
|
|
)
|
|
|
|
|
|
def cloudsql_control_plane_state(
|
|
project: str,
|
|
instance: str,
|
|
expected_host: str,
|
|
expected_private_network: str,
|
|
*,
|
|
timeout: float,
|
|
) -> dict[str, Any]:
|
|
completed = _run(
|
|
[
|
|
"gcloud",
|
|
"sql",
|
|
"instances",
|
|
"describe",
|
|
instance,
|
|
f"--project={project}",
|
|
"--format=json",
|
|
"--quiet",
|
|
],
|
|
timeout=timeout,
|
|
)
|
|
raw = _require_success(completed, "Cloud SQL control-plane readback")
|
|
try:
|
|
payload = json.loads(raw)
|
|
except json.JSONDecodeError as exc:
|
|
raise RestoreError("Cloud SQL control-plane readback returned invalid JSON") from exc
|
|
settings = payload.get("settings") or {}
|
|
ip_configuration = settings.get("ipConfiguration") or {}
|
|
addresses = payload.get("ipAddresses") or []
|
|
private_addresses = sorted(
|
|
str(row.get("ipAddress")) for row in addresses if row.get("type") == "PRIVATE" and row.get("ipAddress")
|
|
)
|
|
private_network = str(ip_configuration.get("privateNetwork") or "")
|
|
non_private_addresses = sorted(
|
|
str(row.get("ipAddress")) for row in addresses if row.get("type") != "PRIVATE" and row.get("ipAddress")
|
|
)
|
|
checks = {
|
|
"instance": payload.get("name") == instance,
|
|
"state": payload.get("state") == "RUNNABLE",
|
|
"postgres_16": str(payload.get("databaseVersion") or "").startswith("POSTGRES_16"),
|
|
"public_ip_disabled": ip_configuration.get("ipv4Enabled") is False,
|
|
"private_network": private_network == expected_private_network,
|
|
"expected_private_host": expected_host in private_addresses,
|
|
"no_non_private_address": not non_private_addresses,
|
|
}
|
|
failed = sorted(name for name, passed in checks.items() if not passed)
|
|
if failed:
|
|
raise RestoreError("Cloud SQL control-plane preflight failed: " + ", ".join(failed))
|
|
return {
|
|
"project": project,
|
|
"instance": instance,
|
|
"database_version": payload.get("databaseVersion"),
|
|
"region": payload.get("region"),
|
|
"state": payload.get("state"),
|
|
"tier": settings.get("tier"),
|
|
"private_network": private_network,
|
|
"expected_private_network": expected_private_network,
|
|
"private_addresses": private_addresses,
|
|
"expected_private_host": expected_host,
|
|
"public_ip_disabled": True,
|
|
"checks": checks,
|
|
}
|
|
|
|
|
|
def gce_metadata_value(path: str, *, timeout: float) -> str:
|
|
request = urllib.request.Request(
|
|
f"{GCE_METADATA_ROOT}/{path}",
|
|
headers={"Metadata-Flavor": "Google"},
|
|
method="GET",
|
|
)
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
try:
|
|
with opener.open(request, timeout=timeout) as response:
|
|
if response.headers.get("Metadata-Flavor") != "Google":
|
|
raise RestoreError("GCE metadata response is missing Metadata-Flavor: Google")
|
|
value = response.read(4096).decode("utf-8", errors="strict").strip()
|
|
except (OSError, UnicodeError, urllib.error.URLError) as exc:
|
|
raise RestoreError(f"GCE metadata readback failed for {path}") from exc
|
|
if not value or any(character in value for character in "\r\n\x00"):
|
|
raise RestoreError(f"GCE metadata readback is invalid for {path}")
|
|
return value
|
|
|
|
|
|
def gce_compute_identity(
|
|
project: str,
|
|
expected_instance: str,
|
|
expected_zone: str,
|
|
expected_private_network: str,
|
|
expected_service_account: str,
|
|
*,
|
|
timeout: float,
|
|
) -> dict[str, Any]:
|
|
metadata = {
|
|
"project": gce_metadata_value("project/project-id", timeout=timeout),
|
|
"project_number": gce_metadata_value("project/numeric-project-id", timeout=timeout),
|
|
"instance": gce_metadata_value("instance/name", timeout=timeout),
|
|
"instance_id": gce_metadata_value("instance/id", timeout=timeout),
|
|
"zone_resource": gce_metadata_value("instance/zone", timeout=timeout),
|
|
"network_resource": gce_metadata_value("instance/network-interfaces/0/network", timeout=timeout),
|
|
"private_ip": gce_metadata_value("instance/network-interfaces/0/ip", timeout=timeout),
|
|
"service_account": gce_metadata_value("instance/service-accounts/default/email", timeout=timeout),
|
|
}
|
|
zone = metadata["zone_resource"].rsplit("/", 1)[-1]
|
|
network_name = expected_private_network.rsplit("/", 1)[-1]
|
|
expected_network_resource = f"projects/{metadata['project_number']}/networks/{network_name}"
|
|
try:
|
|
private_ip = ipaddress.ip_address(metadata["private_ip"])
|
|
except ValueError as exc:
|
|
raise RestoreError("GCE metadata private IP is invalid") from exc
|
|
checks = {
|
|
"project": metadata["project"] == project,
|
|
"project_number": metadata["project_number"].isdigit(),
|
|
"expected_network_project": expected_private_network.startswith(f"projects/{project}/global/networks/"),
|
|
"instance": metadata["instance"] == expected_instance,
|
|
"instance_id": metadata["instance_id"].isdigit(),
|
|
"zone": zone == expected_zone,
|
|
"network": metadata["network_resource"] == expected_network_resource,
|
|
"private_ip": any(
|
|
private_ip in network
|
|
for network in (
|
|
ipaddress.ip_network("10.0.0.0/8"),
|
|
ipaddress.ip_network("172.16.0.0/12"),
|
|
ipaddress.ip_network("192.168.0.0/16"),
|
|
)
|
|
),
|
|
"service_account": metadata["service_account"] == expected_service_account,
|
|
}
|
|
failed = sorted(name for name, passed in checks.items() if not passed)
|
|
if failed:
|
|
raise RestoreError("GCE compute identity preflight failed: " + ", ".join(failed))
|
|
return {
|
|
**metadata,
|
|
"zone": zone,
|
|
"expected_zone": expected_zone,
|
|
"expected_private_network": expected_private_network,
|
|
"expected_network_resource": expected_network_resource,
|
|
"expected_service_account": expected_service_account,
|
|
"checks": checks,
|
|
}
|
|
|
|
|
|
def resolve_password(project: str, secret: str, *, timeout: float) -> str:
|
|
completed = _run(
|
|
[
|
|
"gcloud",
|
|
"secrets",
|
|
"versions",
|
|
"access",
|
|
"latest",
|
|
f"--secret={secret}",
|
|
f"--project={project}",
|
|
"--quiet",
|
|
],
|
|
timeout=timeout,
|
|
)
|
|
password = _require_success(completed, "Cloud SQL credential resolution").strip()
|
|
if not password:
|
|
raise RestoreError("Cloud SQL credential resolved to an empty value")
|
|
return password
|
|
|
|
|
|
def validate_pg_restore_version(
|
|
pg_restore_bin: str,
|
|
source_manifest: dict[str, Any],
|
|
*,
|
|
timeout: float,
|
|
) -> dict[str, Any]:
|
|
completed = _run([pg_restore_bin, "--version"], timeout=timeout)
|
|
raw = _require_success(completed, "pg_restore version preflight").strip()
|
|
match = PG_RESTORE_VERSION_RE.search(raw)
|
|
if match is None:
|
|
raise RestoreError("pg_restore version preflight returned an unrecognized value")
|
|
client_major = int(match.group("major"))
|
|
source_version_num = int(source_manifest["singleton"]["identity"]["server_version_num"])
|
|
source_major = source_version_num // 10000
|
|
if client_major < source_major:
|
|
raise RestoreError(f"pg_restore major {client_major} is older than source PostgreSQL major {source_major}")
|
|
return {
|
|
"pg_restore_bin": pg_restore_bin,
|
|
"pg_restore_version": raw,
|
|
"pg_restore_major": client_major,
|
|
"source_postgres_major": source_major,
|
|
"compatible": True,
|
|
}
|
|
|
|
|
|
def postgres_env(password: str, *, read_only: bool) -> dict[str, str]:
|
|
env = {**os.environ, "PGPASSWORD": password}
|
|
options = ["-c statement_timeout=120000"]
|
|
if read_only:
|
|
options.append("-c default_transaction_read_only=on")
|
|
env["PGOPTIONS"] = " ".join(options)
|
|
return env
|
|
|
|
|
|
def conninfo_value(value: Any) -> str:
|
|
return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'"
|
|
|
|
|
|
def postgres_conninfo(args: argparse.Namespace, database: str) -> str:
|
|
return " ".join(
|
|
(
|
|
f"host={conninfo_value(args.ssl_server_name)}",
|
|
f"hostaddr={conninfo_value(args.host)}",
|
|
"port=5432",
|
|
f"dbname={conninfo_value(database)}",
|
|
"user=postgres",
|
|
"sslmode=verify-full",
|
|
f"sslrootcert={conninfo_value(args.ssl_root_cert)}",
|
|
"connect_timeout=8",
|
|
)
|
|
)
|
|
|
|
|
|
def psql_command(args: argparse.Namespace, database: str) -> list[str]:
|
|
return [
|
|
args.psql_bin,
|
|
postgres_conninfo(args, database),
|
|
"-X",
|
|
"-Atq",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
]
|
|
|
|
|
|
def run_psql(
|
|
args: argparse.Namespace,
|
|
password: str,
|
|
database: str,
|
|
sql: str,
|
|
*,
|
|
read_only: bool,
|
|
timeout: float,
|
|
) -> str:
|
|
completed = _run(
|
|
psql_command(args, database),
|
|
timeout=timeout,
|
|
env=postgres_env(password, read_only=read_only),
|
|
input_text=sql,
|
|
)
|
|
return _require_success(completed, "PostgreSQL command", secrets=(password,))
|
|
|
|
|
|
def database_exists(args: argparse.Namespace, password: str, database: str) -> bool:
|
|
output = run_psql(
|
|
args,
|
|
password,
|
|
"postgres",
|
|
f"select count(*) from pg_database where datname = '{database}';\n",
|
|
read_only=True,
|
|
timeout=args.command_timeout,
|
|
).strip()
|
|
if output not in {"0", "1"}:
|
|
raise RestoreError("database existence readback was invalid")
|
|
return output == "1"
|
|
|
|
|
|
def create_database(
|
|
args: argparse.Namespace,
|
|
password: str,
|
|
source_manifest: dict[str, Any],
|
|
) -> None:
|
|
identity = source_manifest["singleton"]["identity"]
|
|
encoding = str(identity.get("server_encoding") or "")
|
|
collation = str(identity.get("database_collation") or "")
|
|
ctype = str(identity.get("database_ctype") or "")
|
|
if encoding != "UTF8":
|
|
raise RestoreError(f"source encoding is not the reviewed UTF8 value: {encoding!r}")
|
|
if not LOCALE_RE.fullmatch(collation) or not LOCALE_RE.fullmatch(ctype):
|
|
raise RestoreError("source collation or ctype is not a safe locale identifier")
|
|
sql = (
|
|
f'create database "{args.target_db}" with template template0 '
|
|
f"encoding 'UTF8' lc_collate '{collation}' lc_ctype '{ctype}';\n"
|
|
)
|
|
run_psql(
|
|
args,
|
|
password,
|
|
"postgres",
|
|
sql,
|
|
read_only=False,
|
|
timeout=args.command_timeout,
|
|
)
|
|
|
|
|
|
def restore_dump(args: argparse.Namespace, password: str) -> None:
|
|
command = [
|
|
args.pg_restore_bin,
|
|
"-d",
|
|
postgres_conninfo(args, args.target_db),
|
|
"--no-owner",
|
|
"--no-acl",
|
|
"--exit-on-error",
|
|
str(args.dump.resolve()),
|
|
]
|
|
completed = _run(
|
|
command,
|
|
timeout=args.restore_timeout,
|
|
env=postgres_env(password, read_only=False),
|
|
)
|
|
_require_success(completed, "canonical PostgreSQL restore", secrets=(password,))
|
|
|
|
|
|
def capture_target_manifest(args: argparse.Namespace, password: str) -> str:
|
|
command = [*psql_command(args, args.target_db), "-f", str(args.manifest_sql.resolve())]
|
|
completed = _run(
|
|
command,
|
|
timeout=args.manifest_timeout,
|
|
env=postgres_env(password, read_only=True),
|
|
)
|
|
return _require_success(completed, "target parity manifest capture", secrets=(password,))
|
|
|
|
|
|
def validate_private_identity(
|
|
target_manifest: dict[str, Any],
|
|
target_db: str,
|
|
source_compute: str,
|
|
ssl_server_name: str,
|
|
ssl_root_cert_sha256: str,
|
|
) -> dict[str, Any]:
|
|
identity = target_manifest["singleton"]["identity"]
|
|
if identity.get("database") != target_db:
|
|
raise RestoreError("target manifest database does not match the generated clone")
|
|
if identity.get("ssl") is not True:
|
|
raise RestoreError("target manifest did not prove TLS")
|
|
if identity.get("transaction_read_only") != "on":
|
|
raise RestoreError("target manifest was not captured in a read-only transaction")
|
|
try:
|
|
address = ipaddress.ip_address(str(identity.get("server_address") or ""))
|
|
except ValueError as exc:
|
|
raise RestoreError("target manifest server address is invalid") from exc
|
|
private = any(
|
|
address in network
|
|
for network in (
|
|
ipaddress.ip_network("10.0.0.0/8"),
|
|
ipaddress.ip_network("172.16.0.0/12"),
|
|
ipaddress.ip_network("192.168.0.0/16"),
|
|
)
|
|
)
|
|
if not private:
|
|
raise RestoreError("target manifest server address is not RFC1918-private")
|
|
return {
|
|
"target_database": target_db,
|
|
"server_address": str(address),
|
|
"server_port": identity.get("server_port"),
|
|
"ssl": True,
|
|
"ssl_version": identity.get("ssl_version"),
|
|
"sslmode": "verify-full",
|
|
"ssl_server_name": ssl_server_name,
|
|
"ssl_root_cert_sha256": ssl_root_cert_sha256,
|
|
"server_identity_verified": True,
|
|
"private_connectivity": True,
|
|
"transaction_read_only": "on",
|
|
"source_compute": source_compute,
|
|
}
|
|
|
|
|
|
def drop_clone(args: argparse.Namespace, password: str) -> dict[str, Any]:
|
|
if not TARGET_DATABASE_RE.fullmatch(args.target_db):
|
|
raise RestoreError("refusing to drop a database outside teleo_clone_*")
|
|
existed_before = database_exists(args, password, args.target_db)
|
|
if existed_before:
|
|
sql = f"""
|
|
select pg_terminate_backend(pid)
|
|
from pg_stat_activity
|
|
where datname = '{args.target_db}' and pid <> pg_backend_pid();
|
|
drop database "{args.target_db}";
|
|
"""
|
|
run_psql(
|
|
args,
|
|
password,
|
|
"postgres",
|
|
sql,
|
|
read_only=False,
|
|
timeout=args.command_timeout,
|
|
)
|
|
remaining = 1 if database_exists(args, password, args.target_db) else 0
|
|
if remaining:
|
|
raise RestoreError("generated clone remained after cleanup")
|
|
return {"existed_before": existed_before, "clone_database_remaining": remaining}
|
|
|
|
|
|
def rollback_database_state(args: argparse.Namespace, password: str) -> dict[str, Any]:
|
|
output = run_psql(
|
|
args,
|
|
password,
|
|
"postgres",
|
|
f"""
|
|
select jsonb_build_object(
|
|
'exists', (select count(*) = 1 from pg_database where datname = '{ROLLBACK_DATABASE}'),
|
|
'datallowconn', (select datallowconn from pg_database where datname = '{ROLLBACK_DATABASE}'),
|
|
'connections', (select count(*) from pg_stat_activity where datname = '{ROLLBACK_DATABASE}')
|
|
)::text;
|
|
""",
|
|
read_only=True,
|
|
timeout=args.command_timeout,
|
|
)
|
|
rows = [line for line in output.splitlines() if line.strip().startswith("{")]
|
|
if len(rows) != 1:
|
|
raise RestoreError("rollback database state readback was invalid")
|
|
state = json.loads(rows[0])
|
|
if state != {"exists": True, "datallowconn": False, "connections": 0}:
|
|
raise RestoreError("disabled rollback database invariant changed")
|
|
return state
|
|
|
|
|
|
def secure_run_dir(run_root: Path, run_id: str, *, create: bool) -> Path:
|
|
if run_root.is_symlink():
|
|
raise RestoreError("run root must not be a symlink")
|
|
root = run_root.resolve()
|
|
if create:
|
|
root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
os.chmod(root, 0o700)
|
|
if not root.is_dir() or root.is_symlink():
|
|
raise RestoreError("run root is absent or unsafe")
|
|
run_dir = root / run_id
|
|
if create:
|
|
try:
|
|
run_dir.mkdir(mode=0o700)
|
|
except FileExistsError as exc:
|
|
raise RestoreError("run directory already exists") from exc
|
|
if not run_dir.is_dir() or run_dir.is_symlink() or run_dir.resolve().parent != root:
|
|
raise RestoreError("run directory escaped the fixed root or is unsafe")
|
|
os.chmod(run_dir, 0o700)
|
|
return run_dir
|
|
|
|
|
|
def write_private(path: Path, content: str | bytes) -> None:
|
|
encoded = content.encode() if isinstance(content, str) else content
|
|
temporary = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp"
|
|
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
try:
|
|
with os.fdopen(descriptor, "wb") as handle:
|
|
handle.write(encoded)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary, path)
|
|
os.chmod(path, 0o600)
|
|
finally:
|
|
try:
|
|
temporary.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
def run_restore(args: argparse.Namespace) -> dict[str, Any]:
|
|
started = time.monotonic()
|
|
args.ssl_root_cert = validate_regular_file(args.ssl_root_cert, "Cloud SQL server CA")
|
|
ssl_root_cert_sha256 = sha256_file(args.ssl_root_cert)
|
|
run_dir = secure_run_dir(args.run_root, args.run_id, create=True)
|
|
receipt: dict[str, Any] = {
|
|
"artifact": "gcp_generated_postgres_snapshot_restore",
|
|
"generated_at_utc": utc_now(),
|
|
"run_id": args.run_id,
|
|
"target_database": args.target_db,
|
|
"execution_contract": {
|
|
"project": args.project,
|
|
"cloudsql_instance": args.cloudsql_instance,
|
|
"host": args.host,
|
|
"private_network": args.expected_private_network,
|
|
"compute_instance": args.expected_compute_instance,
|
|
"compute_zone": args.expected_compute_zone,
|
|
"restore_service_account": args.expected_restore_service_account,
|
|
"ssl_server_name": args.ssl_server_name,
|
|
"ssl_root_cert_sha256": ssl_root_cert_sha256,
|
|
"source": {
|
|
"ssh_target": args.expected_source_ssh_target,
|
|
"container": args.expected_source_container,
|
|
"database": args.expected_source_database,
|
|
"service": args.expected_source_service,
|
|
},
|
|
"password_secret": args.password_secret,
|
|
"service": args.service,
|
|
},
|
|
"status": "running",
|
|
"phase": "validation",
|
|
"source": {},
|
|
"toolchain": {},
|
|
"target": {},
|
|
"parity": {},
|
|
"live_service": {},
|
|
"rollback_database": {},
|
|
"cleanup": {"required": True, "attempted": False, "clone_database_remaining": None},
|
|
"safety": {
|
|
"live_database_named": False,
|
|
"live_profile_modified": False,
|
|
"live_service_restarted": False,
|
|
"telegram_message_sent": False,
|
|
"secret_persisted": False,
|
|
},
|
|
"error": None,
|
|
}
|
|
password = ""
|
|
creation_attempted = False
|
|
try:
|
|
if os.geteuid() != 0:
|
|
raise RestoreError("restore must run as root on the GCP staging VM")
|
|
args.dump = validate_regular_file(args.dump, "custom dump")
|
|
args.source_manifest = validate_regular_file(args.source_manifest, "source manifest")
|
|
args.manifest_sql = validate_regular_file(args.manifest_sql, "parity manifest SQL")
|
|
if sha256_file(args.manifest_sql) != REVIEWED_MANIFEST_SQL_SHA256:
|
|
raise RestoreError("parity manifest SQL does not match the reviewed SHA-256")
|
|
receipt["source"]["dump"] = validate_custom_dump(args.dump, args.expected_dump_sha256)
|
|
receipt["source"]["manifest"] = {
|
|
"path": str(args.source_manifest),
|
|
"bytes": args.source_manifest.stat().st_size,
|
|
"sha256": sha256_file(args.source_manifest),
|
|
}
|
|
source_manifest = load_manifest(args.source_manifest)
|
|
receipt["source"]["capture_receipt"] = validate_source_snapshot_receipt(
|
|
args.source_receipt,
|
|
expected_receipt_sha256=args.expected_source_receipt_sha256,
|
|
expected_authorization_ref=args.expected_capture_authorization_ref,
|
|
expected_source=receipt["execution_contract"]["source"],
|
|
dump=receipt["source"]["dump"],
|
|
source_manifest_path=args.source_manifest,
|
|
manifest_sql_path=args.manifest_sql,
|
|
source_manifest=source_manifest,
|
|
)
|
|
receipt["toolchain"] = validate_pg_restore_version(
|
|
args.pg_restore_bin,
|
|
source_manifest,
|
|
timeout=args.command_timeout,
|
|
)
|
|
receipt["target"]["compute"] = gce_compute_identity(
|
|
args.project,
|
|
args.expected_compute_instance,
|
|
args.expected_compute_zone,
|
|
args.expected_private_network,
|
|
args.expected_restore_service_account,
|
|
timeout=args.command_timeout,
|
|
)
|
|
receipt["live_service"]["before"] = service_state(args.service, timeout=args.command_timeout)
|
|
validate_service_healthy(receipt["live_service"]["before"], "pre-restore")
|
|
|
|
receipt["phase"] = "cloudsql_control_plane_preflight"
|
|
receipt["target"]["cloudsql"] = cloudsql_control_plane_state(
|
|
args.project,
|
|
args.cloudsql_instance,
|
|
args.host,
|
|
args.expected_private_network,
|
|
timeout=args.command_timeout,
|
|
)
|
|
|
|
receipt["phase"] = "credential_resolution"
|
|
password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout)
|
|
if database_exists(args, password, args.target_db):
|
|
raise RestoreError("target clone already exists; refusing to overwrite it")
|
|
|
|
receipt["phase"] = "database_create"
|
|
creation_attempted = True
|
|
create_database(args, password, source_manifest)
|
|
|
|
receipt["phase"] = "pg_restore"
|
|
restore_started = time.monotonic()
|
|
restore_dump(args, password)
|
|
receipt["target"]["restore_seconds"] = round(time.monotonic() - restore_started, 6)
|
|
|
|
receipt["phase"] = "target_manifest"
|
|
target_manifest_text = capture_target_manifest(args, password)
|
|
target_manifest_path = run_dir / "target-manifest.jsonl"
|
|
write_private(target_manifest_path, target_manifest_text)
|
|
target_manifest = load_manifest(target_manifest_path)
|
|
receipt["target"].update(
|
|
{
|
|
"manifest_path": str(target_manifest_path),
|
|
"manifest_bytes": target_manifest_path.stat().st_size,
|
|
"manifest_sha256": sha256_file(target_manifest_path),
|
|
"connectivity": validate_private_identity(
|
|
target_manifest,
|
|
args.target_db,
|
|
receipt["target"]["compute"]["instance"],
|
|
args.ssl_server_name,
|
|
ssl_root_cert_sha256,
|
|
),
|
|
}
|
|
)
|
|
connectivity = receipt["target"]["connectivity"]
|
|
cloudsql = receipt["target"]["cloudsql"]
|
|
compute = receipt["target"]["compute"]
|
|
connectivity_proof = {
|
|
"artifact": "gcp_private_postgres_connectivity",
|
|
"schema": "livingip.gcpPrivatePostgresConnectivity.v2",
|
|
"generated_at_utc": utc_now(),
|
|
"status": "pass",
|
|
"restore_run_id": args.run_id,
|
|
"target_database": args.target_db,
|
|
"project": cloudsql["project"],
|
|
"cloudsql_instance": cloudsql["instance"],
|
|
"private_network": cloudsql["private_network"],
|
|
"source_compute": compute["instance"],
|
|
"source_compute_instance_id": compute["instance_id"],
|
|
"source_compute_zone": compute["zone"],
|
|
"source_compute_private_ip": compute["private_ip"],
|
|
"server_address": connectivity["server_address"],
|
|
"server_port": connectivity["server_port"],
|
|
"ssl": connectivity["ssl"],
|
|
"ssl_version": connectivity["ssl_version"],
|
|
"sslmode": connectivity["sslmode"],
|
|
"ssl_server_name": connectivity["ssl_server_name"],
|
|
"ssl_root_cert_sha256": connectivity["ssl_root_cert_sha256"],
|
|
"server_identity_verified": connectivity["server_identity_verified"],
|
|
"private_connectivity": connectivity["private_connectivity"],
|
|
"public_ip_disabled": cloudsql["public_ip_disabled"],
|
|
"cloudsql_control_plane_checks": cloudsql["checks"],
|
|
"compute_identity_checks": compute["checks"],
|
|
}
|
|
connectivity_proof_path = run_dir / "gcp-private-connectivity.json"
|
|
write_private(connectivity_proof_path, json.dumps(connectivity_proof, indent=2, sort_keys=True) + "\n")
|
|
receipt["target"]["connectivity_proof"] = {
|
|
"path": str(connectivity_proof_path),
|
|
"sha256": sha256_file(connectivity_proof_path),
|
|
"schema": connectivity_proof["schema"],
|
|
}
|
|
|
|
receipt["phase"] = "parity_compare"
|
|
problems, details = compare_manifests(
|
|
source_manifest,
|
|
target_manifest,
|
|
max_target_query_ms=args.max_target_query_ms,
|
|
max_target_source_ratio=args.max_target_source_ratio,
|
|
)
|
|
receipt["parity"] = {
|
|
"status": "pass" if not problems else "fail",
|
|
"problems": problems,
|
|
"details": details,
|
|
"verifier": "ops.verify_postgres_parity_manifest.compare_manifests",
|
|
}
|
|
if problems:
|
|
raise RestoreError("canonical source/target parity comparison failed")
|
|
|
|
receipt["phase"] = "service_and_rollback_readback"
|
|
receipt["rollback_database"] = rollback_database_state(args, password)
|
|
receipt["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout)
|
|
validate_service_healthy(receipt["live_service"]["after"], "post-restore")
|
|
receipt["live_service"]["unchanged"] = receipt["live_service"]["before"] == receipt["live_service"]["after"]
|
|
if not receipt["live_service"]["unchanged"]:
|
|
raise RestoreError("live GCP service state changed during disposable restore")
|
|
|
|
receipt["status"] = "pass"
|
|
receipt["phase"] = "retained_for_no_send_replay"
|
|
receipt["cleanup"] = {
|
|
"required": True,
|
|
"attempted": False,
|
|
"clone_database_remaining": 1,
|
|
"exact_command": shlex.join(
|
|
[
|
|
"sudo",
|
|
"python3",
|
|
"ops/restore_gcp_generated_postgres_snapshot.py",
|
|
"cleanup",
|
|
"--execute",
|
|
"--target-db",
|
|
args.target_db,
|
|
"--run-id",
|
|
args.run_id,
|
|
"--host",
|
|
args.host,
|
|
"--project",
|
|
args.project,
|
|
"--cloudsql-instance",
|
|
args.cloudsql_instance,
|
|
"--expected-private-network",
|
|
args.expected_private_network,
|
|
"--expected-compute-instance",
|
|
args.expected_compute_instance,
|
|
"--expected-compute-zone",
|
|
args.expected_compute_zone,
|
|
"--expected-restore-service-account",
|
|
args.expected_restore_service_account,
|
|
"--ssl-server-name",
|
|
args.ssl_server_name,
|
|
"--ssl-root-cert",
|
|
str(args.ssl_root_cert),
|
|
"--expected-source-ssh-target",
|
|
args.expected_source_ssh_target,
|
|
"--expected-source-container",
|
|
args.expected_source_container,
|
|
"--expected-source-database",
|
|
args.expected_source_database,
|
|
"--expected-source-service",
|
|
args.expected_source_service,
|
|
"--password-secret",
|
|
args.password_secret,
|
|
"--service",
|
|
args.service,
|
|
]
|
|
),
|
|
}
|
|
except (
|
|
OSError,
|
|
ValueError,
|
|
KeyError,
|
|
TypeError,
|
|
json.JSONDecodeError,
|
|
subprocess.TimeoutExpired,
|
|
RestoreError,
|
|
KeyboardInterrupt,
|
|
) as exc:
|
|
receipt["status"] = "fail"
|
|
receipt["error"] = {
|
|
"phase": receipt.get("phase"),
|
|
"type": type(exc).__name__,
|
|
"message": str(exc)[-2000:],
|
|
}
|
|
if creation_attempted and password:
|
|
receipt["cleanup"]["attempted"] = True
|
|
try:
|
|
receipt["cleanup"].update(drop_clone(args, password))
|
|
except Exception as cleanup_exc: # preserve both failures in the receipt
|
|
receipt["cleanup"]["error"] = str(cleanup_exc)[-2000:]
|
|
elif password:
|
|
try:
|
|
receipt["cleanup"]["clone_database_remaining"] = (
|
|
1 if database_exists(args, password, args.target_db) else 0
|
|
)
|
|
except Exception as cleanup_readback_exc:
|
|
receipt["cleanup"]["readback_error"] = str(cleanup_readback_exc)[-2000:]
|
|
finally:
|
|
if "after" not in receipt["live_service"] and "before" in receipt["live_service"]:
|
|
try:
|
|
receipt["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout)
|
|
receipt["live_service"]["unchanged"] = (
|
|
receipt["live_service"]["before"] == receipt["live_service"]["after"]
|
|
)
|
|
except Exception as service_exc:
|
|
receipt["live_service"]["after_error"] = str(service_exc)[-2000:]
|
|
receipt["duration_seconds"] = round(time.monotonic() - started, 6)
|
|
receipt["completed_at_utc"] = utc_now()
|
|
write_private(run_dir / "restore-receipt.json", json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
|
password = ""
|
|
return receipt
|
|
|
|
|
|
def run_cleanup(args: argparse.Namespace) -> dict[str, Any]:
|
|
args.ssl_root_cert = validate_regular_file(args.ssl_root_cert, "Cloud SQL server CA")
|
|
ssl_root_cert_sha256 = sha256_file(args.ssl_root_cert)
|
|
run_dir = secure_run_dir(args.run_root, args.run_id, create=False)
|
|
restore_receipt_path = run_dir / "restore-receipt.json"
|
|
restore_receipt_path = validate_regular_file(restore_receipt_path, "restore receipt")
|
|
restore_receipt_sha256 = sha256_file(restore_receipt_path)
|
|
restore_receipt = json.loads(restore_receipt_path.read_text(encoding="utf-8"))
|
|
if restore_receipt.get("status") != "pass":
|
|
raise RestoreError("cleanup requires a passing restore receipt")
|
|
if restore_receipt.get("target_database") != args.target_db:
|
|
raise RestoreError("cleanup target does not match the retained restore receipt")
|
|
if restore_receipt.get("run_id") != args.run_id:
|
|
raise RestoreError("cleanup run id does not match the retained restore receipt")
|
|
|
|
expected_contract = {
|
|
"project": args.project,
|
|
"cloudsql_instance": args.cloudsql_instance,
|
|
"host": args.host,
|
|
"private_network": args.expected_private_network,
|
|
"compute_instance": args.expected_compute_instance,
|
|
"compute_zone": args.expected_compute_zone,
|
|
"restore_service_account": args.expected_restore_service_account,
|
|
"ssl_server_name": args.ssl_server_name,
|
|
"ssl_root_cert_sha256": ssl_root_cert_sha256,
|
|
"source": {
|
|
"ssh_target": args.expected_source_ssh_target,
|
|
"container": args.expected_source_container,
|
|
"database": args.expected_source_database,
|
|
"service": args.expected_source_service,
|
|
},
|
|
"password_secret": args.password_secret,
|
|
"service": args.service,
|
|
}
|
|
if restore_receipt.get("execution_contract") != expected_contract:
|
|
raise RestoreError("cleanup execution contract does not match the retained restore receipt")
|
|
restore_cloudsql = (restore_receipt.get("target") or {}).get("cloudsql") or {}
|
|
restore_topology_checks = {
|
|
"project": restore_cloudsql.get("project") == args.project,
|
|
"instance": restore_cloudsql.get("instance") == args.cloudsql_instance,
|
|
"host": restore_cloudsql.get("expected_private_host") == args.host,
|
|
"private_network": restore_cloudsql.get("expected_private_network") == args.expected_private_network,
|
|
"public_ip_disabled": restore_cloudsql.get("public_ip_disabled") is True,
|
|
}
|
|
failed_topology_checks = sorted(name for name, passed in restore_topology_checks.items() if not passed)
|
|
if failed_topology_checks:
|
|
raise RestoreError(
|
|
"cleanup target topology does not match the retained restore receipt: " + ", ".join(failed_topology_checks)
|
|
)
|
|
restore_compute = (restore_receipt.get("target") or {}).get("compute") or {}
|
|
restore_compute_checks = {
|
|
"project": restore_compute.get("project") == args.project,
|
|
"instance": restore_compute.get("instance") == args.expected_compute_instance,
|
|
"zone": restore_compute.get("zone") == args.expected_compute_zone,
|
|
"private_network": restore_compute.get("expected_private_network") == args.expected_private_network,
|
|
"service_account": restore_compute.get("service_account") == args.expected_restore_service_account,
|
|
"checks": bool(restore_compute.get("checks"))
|
|
and all(value is True for value in restore_compute["checks"].values()),
|
|
}
|
|
failed_compute_checks = sorted(name for name, passed in restore_compute_checks.items() if not passed)
|
|
if failed_compute_checks:
|
|
raise RestoreError(
|
|
"cleanup compute identity does not match the retained restore receipt: " + ", ".join(failed_compute_checks)
|
|
)
|
|
|
|
started = time.monotonic()
|
|
payload: dict[str, Any] = {
|
|
"artifact": "gcp_generated_postgres_snapshot_cleanup",
|
|
"generated_at_utc": utc_now(),
|
|
"run_id": args.run_id,
|
|
"target_database": args.target_db,
|
|
"restore_receipt": {"path": str(restore_receipt_path), "sha256": restore_receipt_sha256},
|
|
"execution_contract": expected_contract,
|
|
"target": {},
|
|
"status": "running",
|
|
"phase": "cloudsql_control_plane_preflight",
|
|
"database": {"clone_database_remaining": None},
|
|
"rollback_database": {},
|
|
"live_service": {},
|
|
"safety": {
|
|
"live_database_named": False,
|
|
"live_profile_modified": False,
|
|
"live_service_restarted": False,
|
|
"telegram_message_sent": False,
|
|
"secret_persisted": False,
|
|
},
|
|
"error": None,
|
|
}
|
|
password = ""
|
|
try:
|
|
payload["target"]["cloudsql"] = cloudsql_control_plane_state(
|
|
args.project,
|
|
args.cloudsql_instance,
|
|
args.host,
|
|
args.expected_private_network,
|
|
timeout=args.command_timeout,
|
|
)
|
|
payload["target"]["compute"] = gce_compute_identity(
|
|
args.project,
|
|
args.expected_compute_instance,
|
|
args.expected_compute_zone,
|
|
args.expected_private_network,
|
|
args.expected_restore_service_account,
|
|
timeout=args.command_timeout,
|
|
)
|
|
cleanup_compute_mismatches = sorted(
|
|
field
|
|
for field in (
|
|
"project",
|
|
"project_number",
|
|
"instance",
|
|
"instance_id",
|
|
"zone",
|
|
"network_resource",
|
|
"private_ip",
|
|
"service_account",
|
|
)
|
|
if payload["target"]["compute"].get(field) != restore_compute.get(field)
|
|
)
|
|
if cleanup_compute_mismatches:
|
|
raise RestoreError("cleanup GCE identity changed since restore: " + ", ".join(cleanup_compute_mismatches))
|
|
payload["phase"] = "service_readback_before"
|
|
payload["live_service"]["before"] = service_state(args.service, timeout=args.command_timeout)
|
|
validate_service_healthy(payload["live_service"]["before"], "pre-cleanup")
|
|
payload["phase"] = "credential_resolution"
|
|
password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout)
|
|
payload["phase"] = "database_drop"
|
|
payload["database"] = drop_clone(args, password)
|
|
payload["phase"] = "rollback_readback"
|
|
payload["rollback_database"] = rollback_database_state(args, password)
|
|
payload["phase"] = "service_readback_after"
|
|
payload["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout)
|
|
validate_service_healthy(payload["live_service"]["after"], "post-cleanup")
|
|
payload["live_service"]["unchanged"] = payload["live_service"]["before"] == payload["live_service"]["after"]
|
|
payload["status"] = (
|
|
"pass"
|
|
if payload["live_service"]["unchanged"] and payload["database"]["clone_database_remaining"] == 0
|
|
else "fail"
|
|
)
|
|
payload["phase"] = "complete"
|
|
except (
|
|
OSError,
|
|
ValueError,
|
|
KeyError,
|
|
TypeError,
|
|
json.JSONDecodeError,
|
|
subprocess.TimeoutExpired,
|
|
RestoreError,
|
|
KeyboardInterrupt,
|
|
) as exc:
|
|
payload["status"] = "fail"
|
|
payload["error"] = {
|
|
"phase": payload.get("phase"),
|
|
"type": type(exc).__name__,
|
|
"message": str(exc)[-2000:],
|
|
}
|
|
if password and payload["database"].get("clone_database_remaining") is None:
|
|
try:
|
|
payload["database"]["clone_database_remaining"] = (
|
|
1 if database_exists(args, password, args.target_db) else 0
|
|
)
|
|
except Exception as readback_exc:
|
|
payload["database"]["readback_error"] = str(readback_exc)[-2000:]
|
|
if "before" in payload["live_service"] and "after" not in payload["live_service"]:
|
|
try:
|
|
payload["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout)
|
|
payload["live_service"]["unchanged"] = (
|
|
payload["live_service"]["before"] == payload["live_service"]["after"]
|
|
)
|
|
except Exception as service_exc:
|
|
payload["live_service"]["after_error"] = str(service_exc)[-2000:]
|
|
finally:
|
|
password = ""
|
|
payload["duration_seconds"] = round(time.monotonic() - started, 6)
|
|
payload["completed_at_utc"] = utc_now()
|
|
write_private(run_dir / "cleanup-receipt.json", json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
|
return payload
|
|
|
|
|
|
def add_common_args(parser: argparse.ArgumentParser) -> None:
|
|
parser.add_argument("--execute", action="store_true")
|
|
parser.add_argument("--target-db", required=True)
|
|
parser.add_argument("--run-id", required=True)
|
|
parser.add_argument("--host", default=DEFAULT_HOST)
|
|
parser.add_argument("--project", default=DEFAULT_PROJECT)
|
|
parser.add_argument("--cloudsql-instance", default=DEFAULT_INSTANCE)
|
|
parser.add_argument("--expected-private-network", default=DEFAULT_PRIVATE_NETWORK)
|
|
parser.add_argument("--expected-compute-instance", default=DEFAULT_COMPUTE_INSTANCE)
|
|
parser.add_argument("--expected-compute-zone", default=DEFAULT_COMPUTE_ZONE)
|
|
parser.add_argument("--expected-restore-service-account", default=DEFAULT_RESTORE_SERVICE_ACCOUNT)
|
|
parser.add_argument("--ssl-server-name", required=True)
|
|
parser.add_argument("--ssl-root-cert", required=True, type=Path)
|
|
parser.add_argument("--expected-source-ssh-target", default=DEFAULT_SOURCE_SSH_TARGET)
|
|
parser.add_argument("--expected-source-container", default=DEFAULT_SOURCE_CONTAINER)
|
|
parser.add_argument("--expected-source-database", default=DEFAULT_SOURCE_DATABASE)
|
|
parser.add_argument("--expected-source-service", default=DEFAULT_SOURCE_SERVICE)
|
|
parser.add_argument("--password-secret", default=DEFAULT_SECRET)
|
|
parser.add_argument("--service", default=DEFAULT_SERVICE)
|
|
parser.add_argument("--command-timeout", type=float, default=120.0)
|
|
parser.add_argument("--psql-bin", default="psql")
|
|
parser.set_defaults(run_root=DEFAULT_RUN_ROOT)
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
subparsers = parser.add_subparsers(dest="operation", required=True)
|
|
restore = subparsers.add_parser("restore", help="restore and retain one proven disposable clone")
|
|
add_common_args(restore)
|
|
restore.add_argument("--dump", required=True, type=Path)
|
|
restore.add_argument("--expected-dump-sha256", required=True)
|
|
restore.add_argument("--source-manifest", required=True, type=Path)
|
|
restore.add_argument("--source-receipt", required=True, type=Path)
|
|
restore.add_argument("--expected-source-receipt-sha256", required=True)
|
|
restore.add_argument("--expected-capture-authorization-ref", required=True)
|
|
restore.add_argument("--manifest-sql", default=DEFAULT_MANIFEST_SQL, type=Path)
|
|
restore.add_argument("--pg-restore-bin", default="pg_restore")
|
|
restore.add_argument("--restore-timeout", type=float, default=900.0)
|
|
restore.add_argument("--manifest-timeout", type=float, default=300.0)
|
|
restore.add_argument("--max-target-query-ms", type=float, default=2000.0)
|
|
restore.add_argument("--max-target-source-ratio", type=float, default=20.0)
|
|
|
|
cleanup = subparsers.add_parser("cleanup", help="drop the exact clone named by a passing receipt")
|
|
add_common_args(cleanup)
|
|
args = parser.parse_args(argv)
|
|
|
|
if not args.execute:
|
|
parser.error("--execute is required for Cloud SQL database lifecycle operations")
|
|
if not TARGET_DATABASE_RE.fullmatch(args.target_db):
|
|
parser.error("--target-db must be a bounded lowercase teleo_clone_* identifier")
|
|
if not RUN_ID_RE.fullmatch(args.run_id):
|
|
parser.error("--run-id must match gcp-restore-<8-48 lowercase letters, digits, or hyphens>")
|
|
if not CLOUDSQL_INSTANCE_RE.fullmatch(args.cloudsql_instance):
|
|
parser.error("--cloudsql-instance must be a safe lowercase Cloud SQL instance name")
|
|
if not PRIVATE_NETWORK_RE.fullmatch(args.expected_private_network):
|
|
parser.error("--expected-private-network must be a full projects/.../global/networks/... resource")
|
|
if not args.expected_private_network.startswith(f"projects/{args.project}/global/networks/"):
|
|
parser.error("--expected-private-network project must match --project")
|
|
if not GCE_INSTANCE_RE.fullmatch(args.expected_compute_instance):
|
|
parser.error("--expected-compute-instance must be a safe lowercase GCE instance name")
|
|
if not GCE_ZONE_RE.fullmatch(args.expected_compute_zone):
|
|
parser.error("--expected-compute-zone must be a safe GCE zone")
|
|
if not SERVICE_ACCOUNT_RE.fullmatch(args.expected_restore_service_account):
|
|
parser.error("--expected-restore-service-account must be an exact GCP service-account email")
|
|
if not SAFE_DNS_NAME_RE.fullmatch(args.ssl_server_name):
|
|
parser.error("--ssl-server-name must be a safe DNS name covered by the Cloud SQL server certificate")
|
|
try:
|
|
ssl_root_cert_mode = args.ssl_root_cert.lstat().st_mode
|
|
except OSError:
|
|
parser.error("--ssl-root-cert must be an existing regular file")
|
|
if not stat.S_ISREG(ssl_root_cert_mode) or args.ssl_root_cert.is_symlink():
|
|
parser.error("--ssl-root-cert must be a regular non-symlink file")
|
|
args.ssl_root_cert = args.ssl_root_cert.resolve()
|
|
if not SAFE_SSH_TARGET_RE.fullmatch(args.expected_source_ssh_target):
|
|
parser.error("--expected-source-ssh-target must be a host or user@host without options")
|
|
for value, flag in (
|
|
(args.expected_source_container, "--expected-source-container"),
|
|
(args.expected_source_database, "--expected-source-database"),
|
|
(args.expected_source_service, "--expected-source-service"),
|
|
):
|
|
if not SAFE_NAME_RE.fullmatch(value):
|
|
parser.error(f"{flag} contains unsafe characters")
|
|
try:
|
|
host = ipaddress.ip_address(args.host)
|
|
except ValueError:
|
|
parser.error("--host must be a literal IP address")
|
|
rfc1918_networks = (
|
|
ipaddress.ip_network("10.0.0.0/8"),
|
|
ipaddress.ip_network("172.16.0.0/12"),
|
|
ipaddress.ip_network("192.168.0.0/16"),
|
|
)
|
|
if not any(host in network for network in rfc1918_networks):
|
|
parser.error("--host must be an RFC1918-private address")
|
|
if args.command_timeout <= 0:
|
|
parser.error("--command-timeout must be positive")
|
|
if args.operation == "restore":
|
|
if not SHA256_RE.fullmatch(args.expected_dump_sha256):
|
|
parser.error("--expected-dump-sha256 must be 64 lowercase hexadecimal characters")
|
|
if not SHA256_RE.fullmatch(args.expected_source_receipt_sha256):
|
|
parser.error("--expected-source-receipt-sha256 must be 64 lowercase hexadecimal characters")
|
|
if not AUTHORIZATION_REF_RE.fullmatch(args.expected_capture_authorization_ref):
|
|
parser.error("--expected-capture-authorization-ref must be 8-256 safe metadata characters")
|
|
for value, flag in (
|
|
(args.restore_timeout, "--restore-timeout"),
|
|
(args.manifest_timeout, "--manifest-timeout"),
|
|
(args.max_target_query_ms, "--max-target-query-ms"),
|
|
(args.max_target_source_ratio, "--max-target-source-ratio"),
|
|
):
|
|
if value <= 0:
|
|
parser.error(f"{flag} must be positive")
|
|
return args
|
|
|
|
|
|
def _termination_handler(signum: int, _frame: Any) -> None:
|
|
raise TerminationRequested(f"received {signal.Signals(signum).name}")
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
previous_handlers = {}
|
|
for watched_signal in (signal.SIGINT, signal.SIGTERM):
|
|
previous_handlers[watched_signal] = signal.getsignal(watched_signal)
|
|
signal.signal(watched_signal, _termination_handler)
|
|
try:
|
|
payload = run_restore(args) if args.operation == "restore" else run_cleanup(args)
|
|
finally:
|
|
for watched_signal, previous in previous_handlers.items():
|
|
signal.signal(watched_signal, previous)
|
|
receipt_name = "restore-receipt.json" if args.operation == "restore" else "cleanup-receipt.json"
|
|
receipt_path = args.run_root.resolve() / args.run_id / receipt_name
|
|
print_private_receipt_summary(receipt_path, payload)
|
|
return 0 if payload.get("status") == "pass" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|