#!/usr/bin/env bash set -euo pipefail # Prepare, and optionally execute, a Cloud SQL shadow-restore drill from a Teleo # SQLite backup. Default mode is dry-run: it generates the Postgres import SQL # and proof manifest, but does not upload to GCS or mutate Cloud SQL. PROJECT_ID="${PROJECT_ID:-teleo-501523}" INSTANCE="${INSTANCE:-teleo-pgvector-standby}" DATABASE="${DATABASE:-teleo_kb}" REGION="${REGION:-europe-west6}" BUCKET="${BUCKET:-teleo-501523-prod-backups}" GCS_PREFIX="${GCS_PREFIX:-kb-dumps/cloudsql-restore-drills}" SCHEMA="${SCHEMA:-teleo_restore}" SQLITE_BACKUP="${SQLITE_BACKUP:-}" EXECUTE="${EXECUTE:-0}" OUTPUT_ROOT="${OUTPUT_ROOT:-./outputs/gcp-infra-hardening-20260707/private-cloudsql-restore-drills}" PROOF_ROOT="${PROOF_ROOT:-./outputs/gcp-infra-hardening-20260707/proofs}" TS="${TS:-$(date -u +%Y%m%dT%H%M%SZ)}" if [[ -z "$SQLITE_BACKUP" ]]; then echo "SQLITE_BACKUP is required" >&2 exit 2 fi if [[ ! -f "$SQLITE_BACKUP" ]]; then echo "SQLite backup not found: $SQLITE_BACKUP" >&2 exit 2 fi mkdir -p "$OUTPUT_ROOT" "$PROOF_ROOT" chmod 700 "$OUTPUT_ROOT" WORK_DIR="$OUTPUT_ROOT/gcp-cloudsql-restore-drill-${TS}" mkdir -p "$WORK_DIR" chmod 700 "$WORK_DIR" IMPORT_SQL="$WORK_DIR/teleo-pipeline-postgres-import.sql" SOURCE_SUMMARY="$WORK_DIR/source-summary.json" TARGET_COUNTS_SQL="$WORK_DIR/target-counts.sql" CONVERT_LOG="$WORK_DIR/sqlite-to-postgres-dump.stdout" OBJECT_NAME="${GCS_PREFIX%/}/${TS}/teleo-pipeline-postgres-import.sql" GCS_IMPORT_URI="gs://${BUCKET}/${OBJECT_NAME}" PROOF_JSON="$PROOF_ROOT/gcp-cloudsql-restore-drill-${TS}.json" IMPORT_START_JSON="$WORK_DIR/cloudsql-import-start.json" IMPORT_WAIT_JSON="$WORK_DIR/cloudsql-import-wait.json" GCS_OBJECT_JSON="$WORK_DIR/gcs-import-object.json" python3 ops/sqlite_to_postgres_dump.py \ --sqlite-backup "$SQLITE_BACKUP" \ --output-sql "$IMPORT_SQL" \ --summary-json "$SOURCE_SUMMARY" \ --schema "$SCHEMA" \ --target-counts-sql "$TARGET_COUNTS_SQL" \ > "$CONVERT_LOG" execute_arg=() if [[ "$EXECUTE" == "1" ]]; then execute_arg=(--execute) fi # The helper records source_table_counts in the proof manifest without a large # stdin heredoc, which can deadlock on constrained macOS Bash pipe buffers. python3 ops/build_gcp_cloudsql_restore_proof.py \ --source-summary "$SOURCE_SUMMARY" \ --sqlite-backup "$SQLITE_BACKUP" \ --import-sql "$IMPORT_SQL" \ --target-counts-sql "$TARGET_COUNTS_SQL" \ --proof-json "$PROOF_JSON" \ --project-id "$PROJECT_ID" \ --instance "$INSTANCE" \ --database "$DATABASE" \ --region "$REGION" \ --gcs-import-uri "$GCS_IMPORT_URI" \ "${execute_arg[@]}" if [[ "$EXECUTE" != "1" ]]; then exit 0 fi mark_execute_failure() { local rc="$1" local line="$2" python3 - "$PROOF_JSON" "$rc" "$line" <<'PY' import json import sys from datetime import UTC, datetime from pathlib import Path proof_path = Path(sys.argv[1]) rc = int(sys.argv[2]) line = sys.argv[3] proof = json.loads(proof_path.read_text()) proof["status"] = "execute_failed" proof["execute_failure"] = { "exitcode": rc, "line": line, "recorded_at_utc": datetime.now(UTC).isoformat(), "stderr_files": { "gcs_upload": str(Path(proof["local_private_artifacts"]["postgres_import_sql"]).with_name("gcs-upload.stderr")), }, } proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n") PY } trap 'rc=$?; mark_execute_failure "$rc" "$LINENO"; exit "$rc"' ERR gcloud storage cp "$IMPORT_SQL" "$GCS_IMPORT_URI" \ --project="$PROJECT_ID" \ > "$WORK_DIR/gcs-upload.stdout" \ 2> "$WORK_DIR/gcs-upload.stderr" gcloud storage objects describe "$GCS_IMPORT_URI" \ --project="$PROJECT_ID" \ --format=json \ > "$GCS_OBJECT_JSON" gcloud sql import sql "$INSTANCE" "$GCS_IMPORT_URI" \ --project="$PROJECT_ID" \ --database="$DATABASE" \ --quiet \ --format=json \ > "$IMPORT_START_JSON" python3 - "$PROOF_JSON" "$GCS_OBJECT_JSON" "$IMPORT_START_JSON" <<'PY' import json import sys from pathlib import Path proof_path = Path(sys.argv[1]) object_path = Path(sys.argv[2]) operation_path = Path(sys.argv[3]) proof = json.loads(proof_path.read_text()) gcs_object = json.loads(object_path.read_text()) operation = json.loads(operation_path.read_text() or "{}") proof["status"] = "import_started" proof["gcs_object"] = { "name": gcs_object.get("name"), "bucket": gcs_object.get("bucket"), "generation": gcs_object.get("generation"), "size": gcs_object.get("size"), "crc32c": gcs_object.get("crc32c"), } proof["cloudsql_import_operation"] = { "name": operation.get("name"), "status": operation.get("status"), "operationType": operation.get("operationType"), "targetId": operation.get("targetId"), } proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n") print(json.dumps(proof["cloudsql_import_operation"], indent=2, sort_keys=True)) PY operation_name="$(python3 - "$IMPORT_START_JSON" <<'PY' import json, sys print(json.loads(open(sys.argv[1]).read()).get("name", "")) PY )" if [[ -n "$operation_name" ]]; then gcloud sql operations wait "$operation_name" \ --project="$PROJECT_ID" \ --timeout=3600 \ --format=json \ > "$IMPORT_WAIT_JSON" python3 - "$PROOF_JSON" "$IMPORT_WAIT_JSON" <<'PY' import json import sys from pathlib import Path proof_path = Path(sys.argv[1]) wait_path = Path(sys.argv[2]) proof = json.loads(proof_path.read_text()) operation = json.loads(wait_path.read_text() or "{}") proof["cloudsql_import_wait"] = { "name": operation.get("name"), "status": operation.get("status"), "operationType": operation.get("operationType"), "targetId": operation.get("targetId"), "error": operation.get("error"), } proof["status"] = "import_operation_done" if operation.get("status") == "DONE" and not operation.get("error") else "import_operation_failed" proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n") print(json.dumps({"status": proof["status"], "proof": str(proof_path)}, indent=2, sort_keys=True)) PY fi