717 lines
27 KiB
Python
717 lines
27 KiB
Python
#!/usr/bin/env python3
|
|
"""Restore a VPS canonical snapshot into one disposable private Cloud SQL database.
|
|
|
|
The helper is intentionally narrower than a production cutover tool. It can
|
|
create only a previously absent ``teleo_clone_*`` database, verifies full
|
|
manifest parity, and deletes the clone automatically whenever restore or proof
|
|
fails. A successful clone is retained only for a bounded no-send replay and is
|
|
removed with the explicit ``cleanup`` operation afterward.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
import re
|
|
import signal
|
|
import stat
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
try:
|
|
from .verify_postgres_parity_manifest import compare_manifests, load_manifest
|
|
except ImportError: # pragma: no cover - direct script execution on the GCP VM
|
|
from verify_postgres_parity_manifest import compare_manifests, load_manifest
|
|
|
|
|
|
DEFAULT_HOST = "10.61.0.3"
|
|
DEFAULT_PROJECT = "teleo-501523"
|
|
DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password"
|
|
DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service"
|
|
DEFAULT_RUN_ROOT = Path("/var/lib/teleo-gcp-restore-runs")
|
|
DEFAULT_MANIFEST_SQL = Path(__file__).with_name("postgres_parity_manifest.sql")
|
|
REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a"
|
|
TARGET_DATABASE_RE = re.compile(r"teleo_clone_[a-z0-9][a-z0-9_]{0,50}\Z")
|
|
RUN_ID_RE = re.compile(r"gcp-restore-[a-z0-9][a-z0-9-]{7,47}\Z")
|
|
SHA256_RE = re.compile(r"[0-9a-f]{64}\Z")
|
|
LOCALE_RE = re.compile(r"[A-Za-z0-9_.@-]{1,64}\Z")
|
|
PG_RESTORE_VERSION_RE = re.compile(r"pg_restore \(PostgreSQL\) (?P<major>[0-9]+)(?:\.[0-9]+)*")
|
|
ROLLBACK_DATABASE = "teleo_canonical_pre_20260712t1905z"
|
|
|
|
|
|
class RestoreError(RuntimeError):
|
|
"""Raised when a guarded restore or cleanup invariant fails."""
|
|
|
|
|
|
class TerminationRequested(RestoreError):
|
|
"""Raised after SIGINT/SIGTERM so clone cleanup still runs."""
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(UTC).isoformat()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def validate_regular_file(path: Path, label: str) -> Path:
|
|
try:
|
|
mode = path.lstat().st_mode
|
|
except OSError as exc:
|
|
raise RestoreError(f"{label} is unavailable: {exc}") from exc
|
|
if not stat.S_ISREG(mode) or path.is_symlink():
|
|
raise RestoreError(f"{label} must be a regular non-symlink file")
|
|
return path.resolve()
|
|
|
|
|
|
def validate_custom_dump(path: Path, expected_sha256: str) -> dict[str, Any]:
|
|
path = validate_regular_file(path, "custom dump")
|
|
if path.stat().st_size <= 5:
|
|
raise RestoreError("custom dump is empty or truncated")
|
|
with path.open("rb") as handle:
|
|
if handle.read(5) != b"PGDMP":
|
|
raise RestoreError("custom dump is missing the PGDMP signature")
|
|
actual_sha256 = sha256_file(path)
|
|
if actual_sha256 != expected_sha256:
|
|
raise RestoreError("custom dump SHA-256 does not match --expected-dump-sha256")
|
|
return {"path": str(path), "bytes": path.stat().st_size, "sha256": actual_sha256}
|
|
|
|
|
|
def _run(
|
|
command: list[str],
|
|
*,
|
|
timeout: float,
|
|
env: dict[str, str] | None = None,
|
|
input_text: str | None = None,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
command,
|
|
capture_output=True,
|
|
check=False,
|
|
env=env,
|
|
input=input_text,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
|
|
|
|
def _bounded_error(completed: subprocess.CompletedProcess[str], secrets: tuple[str, ...] = ()) -> str:
|
|
message = (completed.stderr or completed.stdout or "no command output").strip()
|
|
for secret in secrets:
|
|
if secret:
|
|
message = message.replace(secret, "[REDACTED]")
|
|
return message[-2000:]
|
|
|
|
|
|
def _require_success(
|
|
completed: subprocess.CompletedProcess[str],
|
|
action: str,
|
|
*,
|
|
secrets: tuple[str, ...] = (),
|
|
) -> str:
|
|
if completed.returncode != 0:
|
|
raise RestoreError(f"{action} failed (exit {completed.returncode}): {_bounded_error(completed, secrets)}")
|
|
return completed.stdout
|
|
|
|
|
|
def service_state(service: str, *, timeout: float) -> dict[str, Any]:
|
|
completed = _run(
|
|
[
|
|
"systemctl",
|
|
"show",
|
|
service,
|
|
"-p",
|
|
"ActiveState",
|
|
"-p",
|
|
"SubState",
|
|
"-p",
|
|
"MainPID",
|
|
"-p",
|
|
"NRestarts",
|
|
"-p",
|
|
"ExecMainStartTimestamp",
|
|
"--no-pager",
|
|
],
|
|
timeout=timeout,
|
|
)
|
|
raw = _require_success(completed, "live GCP service readback")
|
|
result = {}
|
|
for line in raw.splitlines():
|
|
key, separator, value = line.partition("=")
|
|
if separator:
|
|
result[key] = value
|
|
required = {"ActiveState", "SubState", "MainPID", "NRestarts", "ExecMainStartTimestamp"}
|
|
if required - set(result):
|
|
raise RestoreError("live GCP service readback omitted required fields")
|
|
return result
|
|
|
|
|
|
def resolve_password(project: str, secret: str, *, timeout: float) -> str:
|
|
completed = _run(
|
|
[
|
|
"gcloud",
|
|
"secrets",
|
|
"versions",
|
|
"access",
|
|
"latest",
|
|
f"--secret={secret}",
|
|
f"--project={project}",
|
|
"--quiet",
|
|
],
|
|
timeout=timeout,
|
|
)
|
|
password = _require_success(completed, "Cloud SQL credential resolution").strip()
|
|
if not password:
|
|
raise RestoreError("Cloud SQL credential resolved to an empty value")
|
|
return password
|
|
|
|
|
|
def validate_pg_restore_version(
|
|
pg_restore_bin: str,
|
|
source_manifest: dict[str, Any],
|
|
*,
|
|
timeout: float,
|
|
) -> dict[str, Any]:
|
|
completed = _run([pg_restore_bin, "--version"], timeout=timeout)
|
|
raw = _require_success(completed, "pg_restore version preflight").strip()
|
|
match = PG_RESTORE_VERSION_RE.search(raw)
|
|
if match is None:
|
|
raise RestoreError("pg_restore version preflight returned an unrecognized value")
|
|
client_major = int(match.group("major"))
|
|
source_version_num = int(source_manifest["singleton"]["identity"]["server_version_num"])
|
|
source_major = source_version_num // 10000
|
|
if client_major < source_major:
|
|
raise RestoreError(f"pg_restore major {client_major} is older than source PostgreSQL major {source_major}")
|
|
return {
|
|
"pg_restore_bin": pg_restore_bin,
|
|
"pg_restore_version": raw,
|
|
"pg_restore_major": client_major,
|
|
"source_postgres_major": source_major,
|
|
"compatible": True,
|
|
}
|
|
|
|
|
|
def postgres_env(password: str, *, read_only: bool) -> dict[str, str]:
|
|
env = {**os.environ, "PGPASSWORD": password}
|
|
options = ["-c statement_timeout=120000"]
|
|
if read_only:
|
|
options.append("-c default_transaction_read_only=on")
|
|
env["PGOPTIONS"] = " ".join(options)
|
|
return env
|
|
|
|
|
|
def psql_command(args: argparse.Namespace, database: str) -> list[str]:
|
|
return [
|
|
args.psql_bin,
|
|
f"host={args.host} port=5432 dbname={database} user=postgres sslmode=require connect_timeout=8",
|
|
"-X",
|
|
"-Atq",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
]
|
|
|
|
|
|
def run_psql(
|
|
args: argparse.Namespace,
|
|
password: str,
|
|
database: str,
|
|
sql: str,
|
|
*,
|
|
read_only: bool,
|
|
timeout: float,
|
|
) -> str:
|
|
completed = _run(
|
|
psql_command(args, database),
|
|
timeout=timeout,
|
|
env=postgres_env(password, read_only=read_only),
|
|
input_text=sql,
|
|
)
|
|
return _require_success(completed, "PostgreSQL command", secrets=(password,))
|
|
|
|
|
|
def database_exists(args: argparse.Namespace, password: str, database: str) -> bool:
|
|
output = run_psql(
|
|
args,
|
|
password,
|
|
"postgres",
|
|
f"select count(*) from pg_database where datname = '{database}';\n",
|
|
read_only=True,
|
|
timeout=args.command_timeout,
|
|
).strip()
|
|
if output not in {"0", "1"}:
|
|
raise RestoreError("database existence readback was invalid")
|
|
return output == "1"
|
|
|
|
|
|
def create_database(
|
|
args: argparse.Namespace,
|
|
password: str,
|
|
source_manifest: dict[str, Any],
|
|
) -> None:
|
|
identity = source_manifest["singleton"]["identity"]
|
|
encoding = str(identity.get("server_encoding") or "")
|
|
collation = str(identity.get("database_collation") or "")
|
|
ctype = str(identity.get("database_ctype") or "")
|
|
if encoding != "UTF8":
|
|
raise RestoreError(f"source encoding is not the reviewed UTF8 value: {encoding!r}")
|
|
if not LOCALE_RE.fullmatch(collation) or not LOCALE_RE.fullmatch(ctype):
|
|
raise RestoreError("source collation or ctype is not a safe locale identifier")
|
|
sql = (
|
|
f'create database "{args.target_db}" with template template0 '
|
|
f"encoding 'UTF8' lc_collate '{collation}' lc_ctype '{ctype}';\n"
|
|
)
|
|
run_psql(
|
|
args,
|
|
password,
|
|
"postgres",
|
|
sql,
|
|
read_only=False,
|
|
timeout=args.command_timeout,
|
|
)
|
|
|
|
|
|
def restore_dump(args: argparse.Namespace, password: str) -> None:
|
|
command = [
|
|
args.pg_restore_bin,
|
|
"-d",
|
|
f"host={args.host} port=5432 dbname={args.target_db} user=postgres sslmode=require connect_timeout=8",
|
|
"--no-owner",
|
|
"--no-acl",
|
|
"--exit-on-error",
|
|
str(args.dump.resolve()),
|
|
]
|
|
completed = _run(
|
|
command,
|
|
timeout=args.restore_timeout,
|
|
env=postgres_env(password, read_only=False),
|
|
)
|
|
_require_success(completed, "canonical PostgreSQL restore", secrets=(password,))
|
|
|
|
|
|
def capture_target_manifest(args: argparse.Namespace, password: str) -> str:
|
|
command = [*psql_command(args, args.target_db), "-f", str(args.manifest_sql.resolve())]
|
|
completed = _run(
|
|
command,
|
|
timeout=args.manifest_timeout,
|
|
env=postgres_env(password, read_only=True),
|
|
)
|
|
return _require_success(completed, "target parity manifest capture", secrets=(password,))
|
|
|
|
|
|
def validate_private_identity(target_manifest: dict[str, Any], target_db: str) -> dict[str, Any]:
|
|
identity = target_manifest["singleton"]["identity"]
|
|
if identity.get("database") != target_db:
|
|
raise RestoreError("target manifest database does not match the generated clone")
|
|
if identity.get("ssl") is not True:
|
|
raise RestoreError("target manifest did not prove TLS")
|
|
if identity.get("transaction_read_only") != "on":
|
|
raise RestoreError("target manifest was not captured in a read-only transaction")
|
|
try:
|
|
address = ipaddress.ip_address(str(identity.get("server_address") or ""))
|
|
except ValueError as exc:
|
|
raise RestoreError("target manifest server address is invalid") from exc
|
|
private = any(
|
|
address in network
|
|
for network in (
|
|
ipaddress.ip_network("10.0.0.0/8"),
|
|
ipaddress.ip_network("172.16.0.0/12"),
|
|
ipaddress.ip_network("192.168.0.0/16"),
|
|
)
|
|
)
|
|
if not private:
|
|
raise RestoreError("target manifest server address is not RFC1918-private")
|
|
return {
|
|
"target_database": target_db,
|
|
"server_address": str(address),
|
|
"server_port": identity.get("server_port"),
|
|
"ssl": True,
|
|
"ssl_version": identity.get("ssl_version"),
|
|
"private_connectivity": True,
|
|
"transaction_read_only": "on",
|
|
"source_compute": "teleo-prod-1",
|
|
}
|
|
|
|
|
|
def drop_clone(args: argparse.Namespace, password: str) -> dict[str, Any]:
|
|
if not TARGET_DATABASE_RE.fullmatch(args.target_db):
|
|
raise RestoreError("refusing to drop a database outside teleo_clone_*")
|
|
existed_before = database_exists(args, password, args.target_db)
|
|
if existed_before:
|
|
sql = f"""
|
|
select pg_terminate_backend(pid)
|
|
from pg_stat_activity
|
|
where datname = '{args.target_db}' and pid <> pg_backend_pid();
|
|
drop database "{args.target_db}";
|
|
"""
|
|
run_psql(
|
|
args,
|
|
password,
|
|
"postgres",
|
|
sql,
|
|
read_only=False,
|
|
timeout=args.command_timeout,
|
|
)
|
|
remaining = 1 if database_exists(args, password, args.target_db) else 0
|
|
if remaining:
|
|
raise RestoreError("generated clone remained after cleanup")
|
|
return {"existed_before": existed_before, "clone_database_remaining": remaining}
|
|
|
|
|
|
def rollback_database_state(args: argparse.Namespace, password: str) -> dict[str, Any]:
|
|
output = run_psql(
|
|
args,
|
|
password,
|
|
"postgres",
|
|
f"""
|
|
select jsonb_build_object(
|
|
'exists', (select count(*) = 1 from pg_database where datname = '{ROLLBACK_DATABASE}'),
|
|
'datallowconn', (select datallowconn from pg_database where datname = '{ROLLBACK_DATABASE}'),
|
|
'connections', (select count(*) from pg_stat_activity where datname = '{ROLLBACK_DATABASE}')
|
|
)::text;
|
|
""",
|
|
read_only=True,
|
|
timeout=args.command_timeout,
|
|
)
|
|
rows = [line for line in output.splitlines() if line.strip().startswith("{")]
|
|
if len(rows) != 1:
|
|
raise RestoreError("rollback database state readback was invalid")
|
|
state = json.loads(rows[0])
|
|
if state != {"exists": True, "datallowconn": False, "connections": 0}:
|
|
raise RestoreError("disabled rollback database invariant changed")
|
|
return state
|
|
|
|
|
|
def secure_run_dir(run_root: Path, run_id: str, *, create: bool) -> Path:
|
|
if run_root.is_symlink():
|
|
raise RestoreError("run root must not be a symlink")
|
|
root = run_root.resolve()
|
|
if create:
|
|
root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
os.chmod(root, 0o700)
|
|
if not root.is_dir() or root.is_symlink():
|
|
raise RestoreError("run root is absent or unsafe")
|
|
run_dir = root / run_id
|
|
if create:
|
|
try:
|
|
run_dir.mkdir(mode=0o700)
|
|
except FileExistsError as exc:
|
|
raise RestoreError("run directory already exists") from exc
|
|
if not run_dir.is_dir() or run_dir.is_symlink() or run_dir.resolve().parent != root:
|
|
raise RestoreError("run directory escaped the fixed root or is unsafe")
|
|
os.chmod(run_dir, 0o700)
|
|
return run_dir
|
|
|
|
|
|
def write_private(path: Path, content: str | bytes) -> None:
|
|
encoded = content.encode() if isinstance(content, str) else content
|
|
temporary = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp"
|
|
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
try:
|
|
with os.fdopen(descriptor, "wb") as handle:
|
|
handle.write(encoded)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary, path)
|
|
os.chmod(path, 0o600)
|
|
finally:
|
|
try:
|
|
temporary.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
def run_restore(args: argparse.Namespace) -> dict[str, Any]:
|
|
started = time.monotonic()
|
|
run_dir = secure_run_dir(args.run_root, args.run_id, create=True)
|
|
receipt: dict[str, Any] = {
|
|
"artifact": "gcp_generated_postgres_snapshot_restore",
|
|
"generated_at_utc": utc_now(),
|
|
"run_id": args.run_id,
|
|
"target_database": args.target_db,
|
|
"status": "running",
|
|
"phase": "validation",
|
|
"source": {},
|
|
"toolchain": {},
|
|
"target": {},
|
|
"parity": {},
|
|
"live_service": {},
|
|
"rollback_database": {},
|
|
"cleanup": {"required": True, "attempted": False, "clone_database_remaining": None},
|
|
"safety": {
|
|
"live_database_named": False,
|
|
"live_profile_modified": False,
|
|
"live_service_restarted": False,
|
|
"telegram_message_sent": False,
|
|
"secret_persisted": False,
|
|
},
|
|
"error": None,
|
|
}
|
|
password = ""
|
|
creation_attempted = False
|
|
try:
|
|
if os.geteuid() != 0:
|
|
raise RestoreError("restore must run as root on the GCP staging VM")
|
|
args.dump = validate_regular_file(args.dump, "custom dump")
|
|
args.source_manifest = validate_regular_file(args.source_manifest, "source manifest")
|
|
args.manifest_sql = validate_regular_file(args.manifest_sql, "parity manifest SQL")
|
|
if sha256_file(args.manifest_sql) != REVIEWED_MANIFEST_SQL_SHA256:
|
|
raise RestoreError("parity manifest SQL does not match the reviewed SHA-256")
|
|
receipt["source"]["dump"] = validate_custom_dump(args.dump, args.expected_dump_sha256)
|
|
receipt["source"]["manifest"] = {
|
|
"path": str(args.source_manifest),
|
|
"bytes": args.source_manifest.stat().st_size,
|
|
"sha256": sha256_file(args.source_manifest),
|
|
}
|
|
source_manifest = load_manifest(args.source_manifest)
|
|
receipt["toolchain"] = validate_pg_restore_version(
|
|
args.pg_restore_bin,
|
|
source_manifest,
|
|
timeout=args.command_timeout,
|
|
)
|
|
receipt["live_service"]["before"] = service_state(args.service, timeout=args.command_timeout)
|
|
|
|
receipt["phase"] = "credential_resolution"
|
|
password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout)
|
|
if database_exists(args, password, args.target_db):
|
|
raise RestoreError("target clone already exists; refusing to overwrite it")
|
|
|
|
receipt["phase"] = "database_create"
|
|
creation_attempted = True
|
|
create_database(args, password, source_manifest)
|
|
|
|
receipt["phase"] = "pg_restore"
|
|
restore_started = time.monotonic()
|
|
restore_dump(args, password)
|
|
receipt["target"]["restore_seconds"] = round(time.monotonic() - restore_started, 6)
|
|
|
|
receipt["phase"] = "target_manifest"
|
|
target_manifest_text = capture_target_manifest(args, password)
|
|
target_manifest_path = run_dir / "target-manifest.jsonl"
|
|
write_private(target_manifest_path, target_manifest_text)
|
|
target_manifest = load_manifest(target_manifest_path)
|
|
receipt["target"].update(
|
|
{
|
|
"manifest_path": str(target_manifest_path),
|
|
"manifest_bytes": target_manifest_path.stat().st_size,
|
|
"manifest_sha256": sha256_file(target_manifest_path),
|
|
"connectivity": validate_private_identity(target_manifest, args.target_db),
|
|
}
|
|
)
|
|
|
|
receipt["phase"] = "parity_compare"
|
|
problems, details = compare_manifests(
|
|
source_manifest,
|
|
target_manifest,
|
|
max_target_query_ms=args.max_target_query_ms,
|
|
max_target_source_ratio=args.max_target_source_ratio,
|
|
)
|
|
receipt["parity"] = {
|
|
"status": "pass" if not problems else "fail",
|
|
"problems": problems,
|
|
"details": details,
|
|
"verifier": "ops.verify_postgres_parity_manifest.compare_manifests",
|
|
}
|
|
if problems:
|
|
raise RestoreError("canonical source/target parity comparison failed")
|
|
|
|
receipt["phase"] = "service_and_rollback_readback"
|
|
receipt["rollback_database"] = rollback_database_state(args, password)
|
|
receipt["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout)
|
|
receipt["live_service"]["unchanged"] = receipt["live_service"]["before"] == receipt["live_service"]["after"]
|
|
if not receipt["live_service"]["unchanged"]:
|
|
raise RestoreError("live GCP service state changed during disposable restore")
|
|
|
|
receipt["status"] = "pass"
|
|
receipt["phase"] = "retained_for_no_send_replay"
|
|
receipt["cleanup"] = {
|
|
"required": True,
|
|
"attempted": False,
|
|
"clone_database_remaining": 1,
|
|
"exact_command": (
|
|
f"sudo python3 ops/restore_gcp_generated_postgres_snapshot.py cleanup --execute "
|
|
f"--target-db {args.target_db} --run-id {args.run_id}"
|
|
),
|
|
}
|
|
except (
|
|
OSError,
|
|
ValueError,
|
|
KeyError,
|
|
TypeError,
|
|
json.JSONDecodeError,
|
|
subprocess.TimeoutExpired,
|
|
RestoreError,
|
|
KeyboardInterrupt,
|
|
) as exc:
|
|
receipt["status"] = "fail"
|
|
receipt["error"] = {
|
|
"phase": receipt.get("phase"),
|
|
"type": type(exc).__name__,
|
|
"message": str(exc)[-2000:],
|
|
}
|
|
if creation_attempted and password:
|
|
receipt["cleanup"]["attempted"] = True
|
|
try:
|
|
receipt["cleanup"].update(drop_clone(args, password))
|
|
except Exception as cleanup_exc: # preserve both failures in the receipt
|
|
receipt["cleanup"]["error"] = str(cleanup_exc)[-2000:]
|
|
elif password:
|
|
try:
|
|
receipt["cleanup"]["clone_database_remaining"] = (
|
|
1 if database_exists(args, password, args.target_db) else 0
|
|
)
|
|
except Exception as cleanup_readback_exc:
|
|
receipt["cleanup"]["readback_error"] = str(cleanup_readback_exc)[-2000:]
|
|
finally:
|
|
if "after" not in receipt["live_service"] and "before" in receipt["live_service"]:
|
|
try:
|
|
receipt["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout)
|
|
receipt["live_service"]["unchanged"] = (
|
|
receipt["live_service"]["before"] == receipt["live_service"]["after"]
|
|
)
|
|
except Exception as service_exc:
|
|
receipt["live_service"]["after_error"] = str(service_exc)[-2000:]
|
|
receipt["duration_seconds"] = round(time.monotonic() - started, 6)
|
|
write_private(run_dir / "restore-receipt.json", json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
|
password = ""
|
|
return receipt
|
|
|
|
|
|
def run_cleanup(args: argparse.Namespace) -> dict[str, Any]:
|
|
run_dir = secure_run_dir(args.run_root, args.run_id, create=False)
|
|
restore_receipt_path = run_dir / "restore-receipt.json"
|
|
restore_receipt_path = validate_regular_file(restore_receipt_path, "restore receipt")
|
|
restore_receipt = json.loads(restore_receipt_path.read_text(encoding="utf-8"))
|
|
if restore_receipt.get("status") != "pass":
|
|
raise RestoreError("cleanup requires a passing restore receipt")
|
|
if restore_receipt.get("target_database") != args.target_db:
|
|
raise RestoreError("cleanup target does not match the retained restore receipt")
|
|
if restore_receipt.get("run_id") != args.run_id:
|
|
raise RestoreError("cleanup run id does not match the retained restore receipt")
|
|
|
|
before = service_state(args.service, timeout=args.command_timeout)
|
|
password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout)
|
|
try:
|
|
dropped = drop_clone(args, password)
|
|
rollback = rollback_database_state(args, password)
|
|
finally:
|
|
password = ""
|
|
after = service_state(args.service, timeout=args.command_timeout)
|
|
payload = {
|
|
"artifact": "gcp_generated_postgres_snapshot_cleanup",
|
|
"generated_at_utc": utc_now(),
|
|
"run_id": args.run_id,
|
|
"target_database": args.target_db,
|
|
"status": "pass" if before == after and dropped["clone_database_remaining"] == 0 else "fail",
|
|
"database": dropped,
|
|
"rollback_database": rollback,
|
|
"live_service": {"before": before, "after": after, "unchanged": before == after},
|
|
"safety": {
|
|
"live_database_named": False,
|
|
"live_profile_modified": False,
|
|
"live_service_restarted": False,
|
|
"telegram_message_sent": False,
|
|
"secret_persisted": False,
|
|
},
|
|
}
|
|
write_private(run_dir / "cleanup-receipt.json", json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
|
return payload
|
|
|
|
|
|
def add_common_args(parser: argparse.ArgumentParser) -> None:
|
|
parser.add_argument("--execute", action="store_true")
|
|
parser.add_argument("--target-db", required=True)
|
|
parser.add_argument("--run-id", required=True)
|
|
parser.add_argument("--host", default=DEFAULT_HOST)
|
|
parser.add_argument("--project", default=DEFAULT_PROJECT)
|
|
parser.add_argument("--password-secret", default=DEFAULT_SECRET)
|
|
parser.add_argument("--service", default=DEFAULT_SERVICE)
|
|
parser.add_argument("--command-timeout", type=float, default=120.0)
|
|
parser.add_argument("--psql-bin", default="psql")
|
|
parser.set_defaults(run_root=DEFAULT_RUN_ROOT)
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
subparsers = parser.add_subparsers(dest="operation", required=True)
|
|
restore = subparsers.add_parser("restore", help="restore and retain one proven disposable clone")
|
|
add_common_args(restore)
|
|
restore.add_argument("--dump", required=True, type=Path)
|
|
restore.add_argument("--expected-dump-sha256", required=True)
|
|
restore.add_argument("--source-manifest", required=True, type=Path)
|
|
restore.add_argument("--manifest-sql", default=DEFAULT_MANIFEST_SQL, type=Path)
|
|
restore.add_argument("--pg-restore-bin", default="pg_restore")
|
|
restore.add_argument("--restore-timeout", type=float, default=900.0)
|
|
restore.add_argument("--manifest-timeout", type=float, default=300.0)
|
|
restore.add_argument("--max-target-query-ms", type=float, default=2000.0)
|
|
restore.add_argument("--max-target-source-ratio", type=float, default=20.0)
|
|
|
|
cleanup = subparsers.add_parser("cleanup", help="drop the exact clone named by a passing receipt")
|
|
add_common_args(cleanup)
|
|
args = parser.parse_args(argv)
|
|
|
|
if not args.execute:
|
|
parser.error("--execute is required for Cloud SQL database lifecycle operations")
|
|
if not TARGET_DATABASE_RE.fullmatch(args.target_db):
|
|
parser.error("--target-db must be a bounded lowercase teleo_clone_* identifier")
|
|
if not RUN_ID_RE.fullmatch(args.run_id):
|
|
parser.error("--run-id must match gcp-restore-<8-48 lowercase letters, digits, or hyphens>")
|
|
try:
|
|
host = ipaddress.ip_address(args.host)
|
|
except ValueError:
|
|
parser.error("--host must be a literal IP address")
|
|
rfc1918_networks = (
|
|
ipaddress.ip_network("10.0.0.0/8"),
|
|
ipaddress.ip_network("172.16.0.0/12"),
|
|
ipaddress.ip_network("192.168.0.0/16"),
|
|
)
|
|
if not any(host in network for network in rfc1918_networks):
|
|
parser.error("--host must be an RFC1918-private address")
|
|
if args.command_timeout <= 0:
|
|
parser.error("--command-timeout must be positive")
|
|
if args.operation == "restore":
|
|
if not SHA256_RE.fullmatch(args.expected_dump_sha256):
|
|
parser.error("--expected-dump-sha256 must be 64 lowercase hexadecimal characters")
|
|
for value, flag in (
|
|
(args.restore_timeout, "--restore-timeout"),
|
|
(args.manifest_timeout, "--manifest-timeout"),
|
|
(args.max_target_query_ms, "--max-target-query-ms"),
|
|
(args.max_target_source_ratio, "--max-target-source-ratio"),
|
|
):
|
|
if value <= 0:
|
|
parser.error(f"{flag} must be positive")
|
|
return args
|
|
|
|
|
|
def _termination_handler(signum: int, _frame: Any) -> None:
|
|
raise TerminationRequested(f"received {signal.Signals(signum).name}")
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
previous_handlers = {}
|
|
for watched_signal in (signal.SIGINT, signal.SIGTERM):
|
|
previous_handlers[watched_signal] = signal.getsignal(watched_signal)
|
|
signal.signal(watched_signal, _termination_handler)
|
|
try:
|
|
payload = run_restore(args) if args.operation == "restore" else run_cleanup(args)
|
|
finally:
|
|
for watched_signal, previous in previous_handlers.items():
|
|
signal.signal(watched_signal, previous)
|
|
print(json.dumps(payload, indent=2, sort_keys=True))
|
|
return 0 if payload.get("status") == "pass" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|