236 lines
8.1 KiB
Bash
Executable file
236 lines
8.1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Prepare, and optionally execute, a Cloud SQL shadow-restore drill from a Teleo
|
|
# SQLite backup. Default mode is dry-run: it generates the Postgres import SQL
|
|
# and proof manifest, but does not upload to GCS or mutate Cloud SQL.
|
|
|
|
PROJECT_ID="${PROJECT_ID:-teleo-501523}"
|
|
INSTANCE="${INSTANCE:-teleo-pgvector-standby}"
|
|
DATABASE="${DATABASE:-teleo_kb}"
|
|
REGION="${REGION:-europe-west6}"
|
|
BUCKET="${BUCKET:-teleo-501523-prod-backups}"
|
|
GCS_PREFIX="${GCS_PREFIX:-kb-dumps/cloudsql-restore-drills}"
|
|
SCHEMA="${SCHEMA:-teleo_restore}"
|
|
SQLITE_BACKUP="${SQLITE_BACKUP:-}"
|
|
EXECUTE="${EXECUTE:-0}"
|
|
OUTPUT_ROOT="${OUTPUT_ROOT:-./outputs/gcp-infra-hardening-20260707/private-cloudsql-restore-drills}"
|
|
PROOF_ROOT="${PROOF_ROOT:-./outputs/gcp-infra-hardening-20260707/proofs}"
|
|
TS="${TS:-$(date -u +%Y%m%dT%H%M%SZ)}"
|
|
|
|
if [[ -z "$SQLITE_BACKUP" ]]; then
|
|
echo "SQLITE_BACKUP is required" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [[ ! -f "$SQLITE_BACKUP" ]]; then
|
|
echo "SQLite backup not found: $SQLITE_BACKUP" >&2
|
|
exit 2
|
|
fi
|
|
|
|
mkdir -p "$OUTPUT_ROOT" "$PROOF_ROOT"
|
|
chmod 700 "$OUTPUT_ROOT"
|
|
|
|
WORK_DIR="$OUTPUT_ROOT/gcp-cloudsql-restore-drill-${TS}"
|
|
mkdir -p "$WORK_DIR"
|
|
chmod 700 "$WORK_DIR"
|
|
|
|
IMPORT_SQL="$WORK_DIR/teleo-pipeline-postgres-import.sql"
|
|
SOURCE_SUMMARY="$WORK_DIR/source-summary.json"
|
|
TARGET_COUNTS_SQL="$WORK_DIR/target-counts.sql"
|
|
CONVERT_LOG="$WORK_DIR/sqlite-to-postgres-dump.stdout"
|
|
OBJECT_NAME="${GCS_PREFIX%/}/${TS}/teleo-pipeline-postgres-import.sql"
|
|
GCS_IMPORT_URI="gs://${BUCKET}/${OBJECT_NAME}"
|
|
PROOF_JSON="$PROOF_ROOT/gcp-cloudsql-restore-drill-${TS}.json"
|
|
IMPORT_START_JSON="$WORK_DIR/cloudsql-import-start.json"
|
|
IMPORT_WAIT_JSON="$WORK_DIR/cloudsql-import-wait.json"
|
|
GCS_OBJECT_JSON="$WORK_DIR/gcs-import-object.json"
|
|
|
|
python3 ops/sqlite_to_postgres_dump.py \
|
|
--sqlite-backup "$SQLITE_BACKUP" \
|
|
--output-sql "$IMPORT_SQL" \
|
|
--summary-json "$SOURCE_SUMMARY" \
|
|
--schema "$SCHEMA" \
|
|
--target-counts-sql "$TARGET_COUNTS_SQL" \
|
|
> "$CONVERT_LOG"
|
|
|
|
python3 - "$SOURCE_SUMMARY" "$SQLITE_BACKUP" "$IMPORT_SQL" "$TARGET_COUNTS_SQL" "$PROOF_JSON" "$PROJECT_ID" "$INSTANCE" "$DATABASE" "$REGION" "$GCS_IMPORT_URI" "$EXECUTE" <<'PY'
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
summary_path = Path(sys.argv[1])
|
|
sqlite_backup = Path(sys.argv[2])
|
|
import_sql = Path(sys.argv[3])
|
|
target_counts_sql = Path(sys.argv[4])
|
|
proof_path = Path(sys.argv[5])
|
|
project_id, instance, database, region, gcs_import_uri, execute = sys.argv[6:12]
|
|
summary = json.loads(summary_path.read_text())
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
proof = {
|
|
"artifact": "gcp_cloudsql_restore_drill",
|
|
"generated_at_utc": datetime.now(UTC).isoformat(),
|
|
"mode": "execute" if execute == "1" else "dry_run",
|
|
"status": "prepared" if execute != "1" else "prepared_for_execute",
|
|
"project_id": project_id,
|
|
"region": region,
|
|
"cloudsql_instance": instance,
|
|
"database": database,
|
|
"postgres_schema": summary["schema"],
|
|
"source_sqlite_backup": str(sqlite_backup),
|
|
"source_sqlite_backup_sha256": sha256(sqlite_backup),
|
|
"source_integrity_check": summary["source_integrity_check"],
|
|
"source_table_count": summary["table_count"],
|
|
"source_total_rows": summary["source_total_rows"],
|
|
"source_table_counts": {table["name"]: table["row_count"] for table in summary["tables"]},
|
|
"gcs_import_uri": gcs_import_uri,
|
|
"local_private_artifacts": {
|
|
"postgres_import_sql": str(import_sql),
|
|
"source_summary_json": str(summary_path),
|
|
"target_counts_sql": str(target_counts_sql),
|
|
},
|
|
"planned_commands": [
|
|
f"gcloud storage cp {import_sql} {gcs_import_uri}",
|
|
f"gcloud sql import sql {instance} {gcs_import_uri} --project={project_id} --database={database} --quiet --format=json",
|
|
"run target-counts.sql from a trusted VPC/Cloud SQL connector path and compare source_total_rows/source_table_count",
|
|
],
|
|
"not_proven_by_this_artifact": [
|
|
"GCS object generation until EXECUTE=1 succeeds",
|
|
"Cloud SQL import operation until EXECUTE=1 succeeds",
|
|
"Cloud SQL query readback until target-counts.sql is run from a trusted VPC/connector path",
|
|
],
|
|
}
|
|
proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n")
|
|
print(json.dumps({
|
|
"artifact": proof["artifact"],
|
|
"mode": proof["mode"],
|
|
"status": proof["status"],
|
|
"source_integrity_check": proof["source_integrity_check"],
|
|
"source_table_count": proof["source_table_count"],
|
|
"source_total_rows": proof["source_total_rows"],
|
|
"gcs_import_uri": proof["gcs_import_uri"],
|
|
"proof": str(proof_path),
|
|
}, indent=2, sort_keys=True))
|
|
PY
|
|
|
|
if [[ "$EXECUTE" != "1" ]]; then
|
|
exit 0
|
|
fi
|
|
|
|
mark_execute_failure() {
|
|
local rc="$1"
|
|
local line="$2"
|
|
python3 - "$PROOF_JSON" "$rc" "$line" <<'PY'
|
|
import json
|
|
import sys
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
proof_path = Path(sys.argv[1])
|
|
rc = int(sys.argv[2])
|
|
line = sys.argv[3]
|
|
proof = json.loads(proof_path.read_text())
|
|
proof["status"] = "execute_failed"
|
|
proof["execute_failure"] = {
|
|
"exitcode": rc,
|
|
"line": line,
|
|
"recorded_at_utc": datetime.now(UTC).isoformat(),
|
|
"stderr_files": {
|
|
"gcs_upload": str(Path(proof["local_private_artifacts"]["postgres_import_sql"]).with_name("gcs-upload.stderr")),
|
|
},
|
|
}
|
|
proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n")
|
|
PY
|
|
}
|
|
|
|
trap 'rc=$?; mark_execute_failure "$rc" "$LINENO"; exit "$rc"' ERR
|
|
|
|
gcloud storage cp "$IMPORT_SQL" "$GCS_IMPORT_URI" \
|
|
--project="$PROJECT_ID" \
|
|
> "$WORK_DIR/gcs-upload.stdout" \
|
|
2> "$WORK_DIR/gcs-upload.stderr"
|
|
|
|
gcloud storage objects describe "$GCS_IMPORT_URI" \
|
|
--project="$PROJECT_ID" \
|
|
--format=json \
|
|
> "$GCS_OBJECT_JSON"
|
|
|
|
gcloud sql import sql "$INSTANCE" "$GCS_IMPORT_URI" \
|
|
--project="$PROJECT_ID" \
|
|
--database="$DATABASE" \
|
|
--quiet \
|
|
--format=json \
|
|
> "$IMPORT_START_JSON"
|
|
|
|
python3 - "$PROOF_JSON" "$GCS_OBJECT_JSON" "$IMPORT_START_JSON" <<'PY'
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
proof_path = Path(sys.argv[1])
|
|
object_path = Path(sys.argv[2])
|
|
operation_path = Path(sys.argv[3])
|
|
proof = json.loads(proof_path.read_text())
|
|
gcs_object = json.loads(object_path.read_text())
|
|
operation = json.loads(operation_path.read_text() or "{}")
|
|
proof["status"] = "import_started"
|
|
proof["gcs_object"] = {
|
|
"name": gcs_object.get("name"),
|
|
"bucket": gcs_object.get("bucket"),
|
|
"generation": gcs_object.get("generation"),
|
|
"size": gcs_object.get("size"),
|
|
"crc32c": gcs_object.get("crc32c"),
|
|
}
|
|
proof["cloudsql_import_operation"] = {
|
|
"name": operation.get("name"),
|
|
"status": operation.get("status"),
|
|
"operationType": operation.get("operationType"),
|
|
"targetId": operation.get("targetId"),
|
|
}
|
|
proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n")
|
|
print(json.dumps(proof["cloudsql_import_operation"], indent=2, sort_keys=True))
|
|
PY
|
|
|
|
operation_name="$(python3 - "$IMPORT_START_JSON" <<'PY'
|
|
import json, sys
|
|
print(json.loads(open(sys.argv[1]).read()).get("name", ""))
|
|
PY
|
|
)"
|
|
|
|
if [[ -n "$operation_name" ]]; then
|
|
gcloud sql operations wait "$operation_name" \
|
|
--project="$PROJECT_ID" \
|
|
--timeout=3600 \
|
|
--format=json \
|
|
> "$IMPORT_WAIT_JSON"
|
|
|
|
python3 - "$PROOF_JSON" "$IMPORT_WAIT_JSON" <<'PY'
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
proof_path = Path(sys.argv[1])
|
|
wait_path = Path(sys.argv[2])
|
|
proof = json.loads(proof_path.read_text())
|
|
operation = json.loads(wait_path.read_text() or "{}")
|
|
proof["cloudsql_import_wait"] = {
|
|
"name": operation.get("name"),
|
|
"status": operation.get("status"),
|
|
"operationType": operation.get("operationType"),
|
|
"targetId": operation.get("targetId"),
|
|
"error": operation.get("error"),
|
|
}
|
|
proof["status"] = "import_operation_done" if operation.get("status") == "DONE" and not operation.get("error") else "import_operation_failed"
|
|
proof_path.write_text(json.dumps(proof, indent=2, sort_keys=True) + "\n")
|
|
print(json.dumps({"status": proof["status"], "proof": str(proof_path)}, indent=2, sort_keys=True))
|
|
PY
|
|
fi
|