125 lines
4.8 KiB
Python
125 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Verify Cloud SQL restore readback against a Teleo restore drill proof.
|
|
|
|
This script is intentionally post-import and non-mutating. Run the
|
|
`target-counts.sql` generated by `ops/run_gcp_cloudsql_restore_drill.sh` against
|
|
Cloud SQL with CSV output, then pass that CSV here. The verifier compares table
|
|
counts and total rows with the source counts retained in the drill proof.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
|
|
def load_json(path: Path) -> dict[str, object]:
|
|
return json.loads(path.read_text())
|
|
|
|
|
|
def read_target_counts(path: Path) -> dict[str, int]:
|
|
rows: dict[str, int] = {}
|
|
with path.open(newline="") as handle:
|
|
reader = csv.DictReader(handle)
|
|
required = {"table_name", "row_count"}
|
|
missing = required - set(reader.fieldnames or [])
|
|
if missing:
|
|
raise ValueError(f"target counts CSV missing columns: {sorted(missing)}")
|
|
for row in reader:
|
|
table = (row.get("table_name") or "").strip()
|
|
if not table:
|
|
continue
|
|
rows[table] = int(row.get("row_count") or "0")
|
|
return rows
|
|
|
|
|
|
def source_counts_from_drill(proof: dict[str, object]) -> dict[str, int] | None:
|
|
source_counts = proof.get("source_table_counts")
|
|
if isinstance(source_counts, dict):
|
|
return {str(table): int(count) for table, count in source_counts.items()}
|
|
return None
|
|
|
|
|
|
def compare_counts(source_counts: dict[str, int] | None, target_counts: dict[str, int]) -> list[dict[str, object]]:
|
|
if source_counts is None:
|
|
return []
|
|
all_tables = sorted(set(source_counts) | set(target_counts))
|
|
return [
|
|
{
|
|
"table": table,
|
|
"source_rows": source_counts.get(table),
|
|
"target_rows": target_counts.get(table),
|
|
}
|
|
for table in all_tables
|
|
if source_counts.get(table) != target_counts.get(table)
|
|
]
|
|
|
|
|
|
def build_payload(drill_proof_path: Path, target_counts_csv: Path) -> dict[str, object]:
|
|
proof = load_json(drill_proof_path)
|
|
target_counts = read_target_counts(target_counts_csv)
|
|
source_counts = source_counts_from_drill(proof)
|
|
target_table_count = len(target_counts)
|
|
target_total_rows = sum(target_counts.values())
|
|
source_table_count = int(proof.get("source_table_count") or 0)
|
|
source_total_rows = int(proof.get("source_total_rows") or 0)
|
|
mismatches = compare_counts(source_counts, target_counts)
|
|
|
|
problems = []
|
|
if proof.get("status") not in {"import_operation_done", "readback_verified"}:
|
|
problems.append(f"drill_status={proof.get('status')}")
|
|
if source_table_count != target_table_count:
|
|
problems.append(f"table_count source={source_table_count} target={target_table_count}")
|
|
if source_total_rows != target_total_rows:
|
|
problems.append(f"total_rows source={source_total_rows} target={target_total_rows}")
|
|
if mismatches:
|
|
problems.append(f"mismatched_tables={len(mismatches)}")
|
|
|
|
return {
|
|
"artifact": "gcp_cloudsql_restore_readback_verification",
|
|
"generated_at_utc": datetime.now(UTC).isoformat(),
|
|
"drill_proof": str(drill_proof_path),
|
|
"target_counts_csv": str(target_counts_csv),
|
|
"project_id": proof.get("project_id"),
|
|
"cloudsql_instance": proof.get("cloudsql_instance"),
|
|
"database": proof.get("database"),
|
|
"postgres_schema": proof.get("postgres_schema"),
|
|
"source_sqlite_backup_sha256": proof.get("source_sqlite_backup_sha256"),
|
|
"source_table_count": source_table_count,
|
|
"target_table_count": target_table_count,
|
|
"source_total_rows": source_total_rows,
|
|
"target_total_rows": target_total_rows,
|
|
"mismatched_tables": mismatches,
|
|
"status": "pass" if not problems else "fail",
|
|
"problems": problems,
|
|
"not_proven_by_this_artifact": [
|
|
"application cutover to Cloud SQL",
|
|
"ongoing replication after the readback timestamp",
|
|
"semantic query correctness beyond table row-count parity",
|
|
],
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--drill-proof", required=True, type=Path)
|
|
parser.add_argument("--target-counts-csv", required=True, type=Path)
|
|
parser.add_argument(
|
|
"--output",
|
|
default="outputs/gcp-infra-hardening-20260707/proofs/gcp-cloudsql-restore-readback-verification.json",
|
|
type=Path,
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
payload = build_payload(args.drill_proof, args.target_counts_csv)
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
|
print(json.dumps(payload, indent=2, sort_keys=True))
|
|
return 0 if payload["status"] == "pass" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|