#!/usr/bin/env bash set -euo pipefail # Prepare, and optionally execute, a Cloud SQL shadow-restore drill from a Teleo # SQLite backup. Default mode is dry-run: it generates the Postgres import SQL # and proof manifest, but does not upload to GCS or mutate Cloud SQL. PROJECT_ID="${PROJECT_ID:-teleo-501523}" INSTANCE="${INSTANCE:-teleo-pgvector-standby}" DATABASE="${DATABASE:-teleo_kb}" REGION="${REGION:-europe-west6}" BUCKET="${BUCKET:-teleo-501523-prod-backups}" GCS_PREFIX="${GCS_PREFIX:-kb-dumps/cloudsql-restore-drills}" SCHEMA="${SCHEMA:-teleo_restore}" SQLITE_BACKUP="${SQLITE_BACKUP:-}" EXECUTE="${EXECUTE:-0}" OUTPUT_ROOT="${OUTPUT_ROOT:-./outputs/gcp-infra-hardening-20260707/private-cloudsql-restore-drills}" PROOF_ROOT="${PROOF_ROOT:-./outputs/gcp-infra-hardening-20260707/proofs}" TS="${TS:-$(date -u +%Y%m%dT%H%M%SZ)}" if [[ -z "$SQLITE_BACKUP" ]]; then echo "SQLITE_BACKUP is required" >&2 exit 2 fi if [[ ! -f "$SQLITE_BACKUP" ]]; then echo "SQLite backup not found: $SQLITE_BACKUP" >&2 exit 2 fi mkdir -p "$OUTPUT_ROOT" "$PROOF_ROOT" chmod 700 "$OUTPUT_ROOT" WORK_DIR="$OUTPUT_ROOT/gcp-cloudsql-restore-drill-${TS}" mkdir -p "$WORK_DIR" chmod 700 "$WORK_DIR" IMPORT_SQL="$WORK_DIR/teleo-pipeline-postgres-import.sql" SOURCE_SUMMARY="$WORK_DIR/source-summary.json" TARGET_COUNTS_SQL="$WORK_DIR/target-counts.sql" CONVERT_LOG="$WORK_DIR/sqlite-to-postgres-dump.stdout" OBJECT_NAME="${GCS_PREFIX%/}/${TS}/teleo-pipeline-postgres-import.sql" GCS_IMPORT_URI="gs://${BUCKET}/${OBJECT_NAME}" PROOF_JSON="$PROOF_ROOT/gcp-cloudsql-restore-drill-${TS}.json" IMPORT_START_JSON="$WORK_DIR/cloudsql-import-start.json" IMPORT_WAIT_JSON="$WORK_DIR/cloudsql-import-wait.json" GCS_OBJECT_JSON="$WORK_DIR/gcs-import-object.json" python3 ops/sqlite_to_postgres_dump.py \ --sqlite-backup "$SQLITE_BACKUP" \ --output-sql "$IMPORT_SQL" \ --summary-json "$SOURCE_SUMMARY" \ --schema "$SCHEMA" \ --target-counts-sql "$TARGET_COUNTS_SQL" \ > "$CONVERT_LOG" python3 - "$SOURCE_SUMMARY" "$SQLITE_BACKUP" "$IMPORT_SQL" "$TARGET_COUNTS_SQL" "$PROOF_JSON" "$PROJECT_ID" "$INSTANCE" "$DATABASE" "$REGION" "$GCS_IMPORT_URI" "$EXECUTE" <<'PY' import hashlib import json import sys from datetime import UTC, datetime from pathlib import Path summary_path = Path(sys.argv[1]) sqlite_backup = Path(sys.argv[2]) import_sql = Path(sys.argv[3]) target_counts_sql = Path(sys.argv[4]) proof_path = Path(sys.argv[5]) project_id, instance, database, region, gcs_import_uri, execute = sys.argv[6:12] summary = json.loads(summary_path.read_text()) def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() proof = { "artifact": "gcp_cloudsql_restore_drill", "generated_at_utc": datetime.now(UTC).isoformat(), "mode": "execute" if execute == "1" else "dry_run", "status": "prepared" if execute != "1" else "prepared_for_execute", "project_id": project_id, "region": region, "cloudsql_instance": instance, "database": database, "postgres_schema": summary["schema"], "source_sqlite_backup": str(sqlite_backup), "source_sqlite_backup_sha256": sha256(sqlite_backup), "source_integrity_check": summary["source_integrity_check"], "source_table_count": summary["table_count"], "source_total_rows": summary["source_total_rows"], "source_table_counts": {table["name"]: table["row_count"] for table in summary["tables"]}, "gcs_import_uri": gcs_import_uri, "local_private_artifacts": { "postgres_import_sql": str(import_sql), "source_summary_json": str(summary_path), "target_counts_sql": str(target_counts_sql), }, "planned_commands": [ f"gcloud storage cp {import_sql} {gcs_import_uri}", f"gcloud sql import sql {instance} {gcs_import_uri} --project={project_id} --database={database} --quiet --format=json", "run target-counts.sql from a trusted VPC/Cloud SQL connector path and compare source_total_rows/source_table_count", ], "not_proven_by_this_artifact": [ "GCS object generation until EXECUTE=1 succeeds", "Cloud SQL import operation until EXECUTE=1 succeeds", "Cloud SQL query readback until target-counts.sql is run from a trusted VPC/connector path", ], } proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n") print(json.dumps({ "artifact": proof["artifact"], "mode": proof["mode"], "status": proof["status"], "source_integrity_check": proof["source_integrity_check"], "source_table_count": proof["source_table_count"], "source_total_rows": proof["source_total_rows"], "gcs_import_uri": proof["gcs_import_uri"], "proof": str(proof_path), }, indent=2, sort_keys=True)) PY if [[ "$EXECUTE" != "1" ]]; then exit 0 fi mark_execute_failure() { local rc="$1" local line="$2" python3 - "$PROOF_JSON" "$rc" "$line" <<'PY' import json import sys from datetime import UTC, datetime from pathlib import Path proof_path = Path(sys.argv[1]) rc = int(sys.argv[2]) line = sys.argv[3] proof = json.loads(proof_path.read_text()) proof["status"] = "execute_failed" proof["execute_failure"] = { "exitcode": rc, "line": line, "recorded_at_utc": datetime.now(UTC).isoformat(), "stderr_files": { "gcs_upload": str(Path(proof["local_private_artifacts"]["postgres_import_sql"]).with_name("gcs-upload.stderr")), }, } proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n") PY } trap 'rc=$?; mark_execute_failure "$rc" "$LINENO"; exit "$rc"' ERR gcloud storage cp "$IMPORT_SQL" "$GCS_IMPORT_URI" \ --project="$PROJECT_ID" \ > "$WORK_DIR/gcs-upload.stdout" \ 2> "$WORK_DIR/gcs-upload.stderr" gcloud storage objects describe "$GCS_IMPORT_URI" \ --project="$PROJECT_ID" \ --format=json \ > "$GCS_OBJECT_JSON" gcloud sql import sql "$INSTANCE" "$GCS_IMPORT_URI" \ --project="$PROJECT_ID" \ --database="$DATABASE" \ --quiet \ --format=json \ > "$IMPORT_START_JSON" python3 - "$PROOF_JSON" "$GCS_OBJECT_JSON" "$IMPORT_START_JSON" <<'PY' import json import sys from pathlib import Path proof_path = Path(sys.argv[1]) object_path = Path(sys.argv[2]) operation_path = Path(sys.argv[3]) proof = json.loads(proof_path.read_text()) gcs_object = json.loads(object_path.read_text()) operation = json.loads(operation_path.read_text() or "{}") proof["status"] = "import_started" proof["gcs_object"] = { "name": gcs_object.get("name"), "bucket": gcs_object.get("bucket"), "generation": gcs_object.get("generation"), "size": gcs_object.get("size"), "crc32c": gcs_object.get("crc32c"), } proof["cloudsql_import_operation"] = { "name": operation.get("name"), "status": operation.get("status"), "operationType": operation.get("operationType"), "targetId": operation.get("targetId"), } proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n") print(json.dumps(proof["cloudsql_import_operation"], indent=2, sort_keys=True)) PY operation_name="$(python3 - "$IMPORT_START_JSON" <<'PY' import json, sys print(json.loads(open(sys.argv[1]).read()).get("name", "")) PY )" if [[ -n "$operation_name" ]]; then gcloud sql operations wait "$operation_name" \ --project="$PROJECT_ID" \ --timeout=3600 \ --format=json \ > "$IMPORT_WAIT_JSON" python3 - "$PROOF_JSON" "$IMPORT_WAIT_JSON" <<'PY' import json import sys from pathlib import Path proof_path = Path(sys.argv[1]) wait_path = Path(sys.argv[2]) proof = json.loads(proof_path.read_text()) operation = json.loads(wait_path.read_text() or "{}") proof["cloudsql_import_wait"] = { "name": operation.get("name"), "status": operation.get("status"), "operationType": operation.get("operationType"), "targetId": operation.get("targetId"), "error": operation.get("error"), } proof["status"] = "import_operation_done" if operation.get("status") == "DONE" and not operation.get("error") else "import_operation_failed" proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n") print(json.dumps({"status": proof["status"], "proof": str(proof_path)}, indent=2, sort_keys=True)) PY fi