792 lines
32 KiB
Bash
Executable file
792 lines
32 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
|
|
readonly DISPATCHER_VERSION="teleo-gcp-iap-operator-v1"
|
|
readonly EXPECTED_WORKFLOW_REF="living-ip/teleo-infrastructure/.github/workflows/gcp-iap-operator.yml@refs/heads/main"
|
|
readonly PROJECT_ID="teleo-501523"
|
|
readonly INSTANCE="teleo-prod-1"
|
|
readonly ZONE="europe-west6-a"
|
|
readonly INSTANCE_INTERNAL_IP="10.60.0.3"
|
|
readonly CLOUDSQL_HOST="10.61.0.3"
|
|
readonly CLOUDSQL_SECRET="gcp-teleo-pgvector-standby-postgres-password"
|
|
readonly REMOTE_DISPATCHER="/usr/local/sbin/teleo-gcp-iap-operator"
|
|
readonly REMOTE_RUN_ROOT="/var/lib/teleo-gcp-iap-operator/runs"
|
|
readonly LIVE_PROFILE="/home/teleo/.hermes/profiles/leoclean"
|
|
readonly LIVE_SERVICE="leoclean-gcp-prod-parallel.service"
|
|
readonly STATUS_TARGET="teleo_clone_status"
|
|
readonly CORY_REPLAY_DB="teleo_clone_cory_20260712t1940z"
|
|
readonly CORY_REPLAY_FINGERPRINT="48ea5332c4f303dea02d503447f1ac8e1d8aa00e60d198a566d628b37dbdac1c"
|
|
readonly CORY_REPLAY_LEGACY_DIR="/home/teleo/gcp-cory-model-replay-20260712t1940z"
|
|
readonly DISABLED_ROLLBACK_DB="teleo_canonical_pre_20260712t1905z"
|
|
readonly REQUEST_ID_RE='^iap-[a-z0-9]{12,32}$'
|
|
readonly TARGET_DB_RE='^teleo_clone_[a-z0-9][a-z0-9_]{0,50}$'
|
|
readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
readonly RESULT_HELPER="$SCRIPT_DIR/gcp_iap_operator_result.py"
|
|
|
|
RUNNER_TEMP_DIR=""
|
|
VALIDATED_BUNDLE_DIR=""
|
|
VALIDATED_BUNDLE_SHA256=""
|
|
VALIDATED_BUNDLE_COMMIT=""
|
|
|
|
die() {
|
|
printf 'gcp_iap_operator: %s\n' "$1" >&2
|
|
exit 2
|
|
}
|
|
|
|
cleanup_runner_temp() {
|
|
if [[ -n "$RUNNER_TEMP_DIR" && -d "$RUNNER_TEMP_DIR" && "$RUNNER_TEMP_DIR" == "${RUNNER_TEMP:-}/"* ]]; then
|
|
rm -rf -- "$RUNNER_TEMP_DIR"
|
|
fi
|
|
}
|
|
|
|
validate_inputs() {
|
|
local operation="$1"
|
|
local request_id="$2"
|
|
local target_db="$3"
|
|
|
|
case "$operation" in
|
|
status | direct-claim-replay | cleanup-clone) ;;
|
|
*) die "operation is not allowlisted" ;;
|
|
esac
|
|
[[ "$request_id" =~ $REQUEST_ID_RE ]] || die "request_id does not match the required format"
|
|
[[ "$target_db" =~ $TARGET_DB_RE ]] || die "target_db must be a bounded teleo_clone_* identifier"
|
|
if [[ "$operation" == "status" && "$target_db" != "$STATUS_TARGET" ]]; then
|
|
die "status requires the fixed teleo_clone_status sentinel"
|
|
fi
|
|
if [[ "$operation" != "status" && "$target_db" == "$STATUS_TARGET" ]]; then
|
|
die "clone operations cannot use the status sentinel"
|
|
fi
|
|
}
|
|
|
|
emit_json() {
|
|
local operation="$1"
|
|
local request_id="$2"
|
|
local target_db="$3"
|
|
local status="$4"
|
|
shift 4
|
|
python3 - "$operation" "$request_id" "$target_db" "$status" "$@" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
operation, request_id, target_db, status, *pairs = sys.argv[1:]
|
|
payload = {
|
|
"schema": "livingip.gcpIapOperatorRemoteResult.v1",
|
|
"operation": operation,
|
|
"request_id": request_id,
|
|
"target_db": target_db,
|
|
"status": status,
|
|
}
|
|
for pair in pairs:
|
|
key, separator, value = pair.partition("=")
|
|
if not separator:
|
|
raise SystemExit("invalid result field")
|
|
if key in {
|
|
"nrestarts",
|
|
"result_bytes",
|
|
"clone_database_remaining",
|
|
"run_directory_remaining",
|
|
"legacy_directory_remaining",
|
|
"export_file_remaining",
|
|
"leftover_temp_directory_count",
|
|
"rollback_database_connections",
|
|
}:
|
|
payload[key] = int(value)
|
|
elif key in {
|
|
"no_message_send",
|
|
"live_service_unchanged",
|
|
"live_profile_unchanged",
|
|
"full_result_export_ready",
|
|
"rollback_database_datallowconn",
|
|
}:
|
|
payload[key] = value == "true"
|
|
else:
|
|
payload[key] = value
|
|
print(json.dumps(payload, sort_keys=True))
|
|
PY
|
|
}
|
|
|
|
remote_status() {
|
|
local request_id="$1"
|
|
local target_db="$2"
|
|
local active_state sub_state nrestarts dispatcher_hash dispatcher_state
|
|
|
|
[[ -x "$REMOTE_DISPATCHER" && ! -L "$REMOTE_DISPATCHER" ]] || die "root-owned remote dispatcher is absent"
|
|
dispatcher_state="$(stat -c '%U:%G:%a' "$REMOTE_DISPATCHER")"
|
|
[[ "$dispatcher_state" == "root:root:755" ]] || die "remote dispatcher ownership or mode drifted"
|
|
dispatcher_hash="$(sha256sum "$REMOTE_DISPATCHER" | awk '{print $1}')"
|
|
[[ "$dispatcher_hash" =~ ^[0-9a-f]{64}$ ]] || die "remote dispatcher hash readback was invalid"
|
|
active_state="$(systemctl show "$LIVE_SERVICE" --property=ActiveState --value)"
|
|
sub_state="$(systemctl show "$LIVE_SERVICE" --property=SubState --value)"
|
|
nrestarts="$(systemctl show "$LIVE_SERVICE" --property=NRestarts --value)"
|
|
[[ "$active_state" =~ ^[a-z]+$ ]] || die "service ActiveState readback was invalid"
|
|
[[ "$sub_state" =~ ^[a-z-]+$ ]] || die "service SubState readback was invalid"
|
|
[[ "$nrestarts" =~ ^[0-9]+$ ]] || die "service NRestarts readback was invalid"
|
|
|
|
emit_json \
|
|
"status" "$request_id" "$target_db" "pass" \
|
|
"instance=$INSTANCE" \
|
|
"expected_internal_ip=$INSTANCE_INTERNAL_IP" \
|
|
"active_state=$active_state" \
|
|
"sub_state=$sub_state" \
|
|
"nrestarts=$nrestarts" \
|
|
"dispatcher_version=$DISPATCHER_VERSION" \
|
|
"dispatcher_sha256=$dispatcher_hash"
|
|
}
|
|
|
|
require_root() {
|
|
[[ "$EUID" -eq 0 ]] || die "clone operations require the isolated OS Admin identity"
|
|
[[ -n "${SUDO_USER:-}" && "$SUDO_USER" != "root" ]] || die "clone operation lacks a distinct OS Login caller"
|
|
}
|
|
|
|
parity_receipt_relative() {
|
|
local target_db="$1"
|
|
printf 'docs/reports/leo-working-state-20260709/%s-canonical-parity-receipt.json\n' "$target_db"
|
|
}
|
|
|
|
write_run_marker() {
|
|
local marker="$1"
|
|
local request_id="$2"
|
|
local target_db="$3"
|
|
printf 'request_id=%s\ntarget_db=%s\n' "$request_id" "$target_db" >"$marker"
|
|
chmod 0600 "$marker"
|
|
}
|
|
|
|
validate_run_dir() {
|
|
local run_dir="$1"
|
|
local request_id="$2"
|
|
local target_db="$3"
|
|
local marker="$run_dir/.run-owner"
|
|
local resolved expected
|
|
|
|
[[ -d "$run_dir" && ! -L "$run_dir" ]] || die "exact run-owned directory is absent or unsafe"
|
|
resolved="$(realpath "$run_dir")"
|
|
expected="$REMOTE_RUN_ROOT/$request_id"
|
|
[[ "$resolved" == "$expected" ]] || die "run-owned directory escaped its fixed root"
|
|
[[ "$(stat -c '%U:%G:%a' "$run_dir")" == "root:root:700" ]] || die "run directory ownership or mode drifted"
|
|
[[ -f "$marker" && ! -L "$marker" ]] || die "run ownership marker is absent or unsafe"
|
|
[[ "$(cat "$marker")" == $'request_id='"$request_id"$'\ntarget_db='"$target_db" ]] || {
|
|
die "run ownership marker does not match request_id and target_db"
|
|
}
|
|
}
|
|
|
|
validate_replay_temp_root() {
|
|
local temp_path="$1"
|
|
local request_id="$2"
|
|
local temp_resolved temp_state temp_marker temp_marker_state temp_marker_content
|
|
|
|
[[ -d "$temp_path" && ! -L "$temp_path" ]] || die "request-derived temporary replay path is unsafe"
|
|
temp_resolved="$(realpath "$temp_path")"
|
|
[[ "$temp_resolved" == "/tmp/leo-clone-checkpoint-$request_id-gcp-direct-claim-"* ]] || {
|
|
die "request-derived temporary replay path drifted"
|
|
}
|
|
temp_state="$(stat -c '%U:%G:%a' "$temp_path")"
|
|
[[ "$temp_state" == "root:root:700" ]] || die "request-derived temporary replay path ownership or mode drifted"
|
|
temp_marker="$temp_path/.teleo-temp-owner"
|
|
[[ -f "$temp_marker" && ! -L "$temp_marker" ]] || die "temporary replay ownership marker is absent or unsafe"
|
|
temp_marker_state="$(stat -c '%U:%G:%a' "$temp_marker")"
|
|
[[ "$temp_marker_state" == "root:root:600" ]] || die "temporary replay ownership marker state drifted"
|
|
temp_marker_content="$(cat "$temp_marker")"
|
|
[[ "$temp_marker_content" == "$request_id"$'\n'"gcp-direct-claim" ]] || {
|
|
die "temporary replay ownership marker does not match request_id and prompt"
|
|
}
|
|
}
|
|
|
|
remote_upload_path() {
|
|
local request_id="$1"
|
|
local passwd_entry home resolved upload
|
|
|
|
passwd_entry="$(getent passwd "$SUDO_USER")"
|
|
[[ -n "$passwd_entry" ]] || die "OS Login caller has no passwd entry"
|
|
home="$(cut -d: -f6 <<<"$passwd_entry")"
|
|
[[ "$home" == /* && -d "$home" && ! -L "$home" ]] || die "OS Login caller home is unsafe"
|
|
resolved="$(realpath "$home")"
|
|
[[ "$resolved" == "$home" ]] || die "OS Login caller home path drifted"
|
|
upload="$home/.teleo-iap-upload-$request_id.tar.gz"
|
|
[[ -f "$upload" && ! -L "$upload" ]] || die "request-derived bundle upload is absent or unsafe"
|
|
[[ "$(stat -c '%U:%a' "$upload")" == "$SUDO_USER:600" ]] || die "bundle upload owner or mode is unsafe"
|
|
printf '%s\n' "$upload"
|
|
}
|
|
|
|
validate_and_extract_bundle() {
|
|
local archive="$1"
|
|
local bundle_dir="$2"
|
|
local operation="$3"
|
|
local request_id="$4"
|
|
local target_db="$5"
|
|
|
|
python3 - \
|
|
"$archive" "$bundle_dir" "$operation" "$request_id" "$target_db" \
|
|
"$REMOTE_DISPATCHER" "$DISPATCHER_VERSION" "$EXPECTED_WORKFLOW_REF" <<'PY'
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import tarfile
|
|
from pathlib import Path
|
|
|
|
archive_path = Path(sys.argv[1])
|
|
bundle_dir = Path(sys.argv[2]).resolve()
|
|
operation, request_id, target_db = sys.argv[3:6]
|
|
dispatcher_path = Path(sys.argv[6])
|
|
dispatcher_version = sys.argv[7]
|
|
expected_workflow_ref = sys.argv[8]
|
|
common = ["scripts/gcp_iap_operator.sh"]
|
|
direct_claim = [
|
|
"scripts/run_gcp_generated_db_direct_claim_suite.py",
|
|
"scripts/run_leo_clone_bound_handler_checkpoint.py",
|
|
"scripts/working_leo_open_ended_benchmark.py",
|
|
"hermes-agent/leoclean-bin/cloudsql_memory_tool.py",
|
|
"ops/postgres_parity_manifest.sql",
|
|
f"docs/reports/leo-working-state-20260709/{target_db}-canonical-parity-receipt.json",
|
|
]
|
|
expected = common + (direct_claim if operation == "direct-claim-replay" else [])
|
|
if operation not in {"direct-claim-replay", "cleanup-clone"}:
|
|
raise SystemExit("unexpected bundle operation")
|
|
if archive_path.stat().st_size > 5 * 1024 * 1024:
|
|
raise SystemExit("bundle archive exceeds size ceiling")
|
|
|
|
with tarfile.open(archive_path, mode="r:gz") as archive:
|
|
members = archive.getmembers()
|
|
expected_members = sorted([*expected, "bundle-manifest.json"])
|
|
names = [member.name for member in members]
|
|
if sorted(names) != expected_members or len(names) != len(set(names)):
|
|
raise SystemExit("bundle member allowlist mismatch")
|
|
if any(not member.isfile() or member.size > 2 * 1024 * 1024 for member in members):
|
|
raise SystemExit("bundle contains a link, special entry, or oversized file")
|
|
manifest_member = archive.getmember("bundle-manifest.json")
|
|
if manifest_member.size > 64 * 1024:
|
|
raise SystemExit("bundle manifest exceeds size ceiling")
|
|
manifest = json.loads(archive.extractfile(manifest_member).read())
|
|
required = {
|
|
"schema": "livingip.gcpIapOperatorBundle.v1",
|
|
"dispatcher_version": dispatcher_version,
|
|
"operation": operation,
|
|
"request_id": request_id,
|
|
"target_db": target_db,
|
|
"ref": "refs/heads/main",
|
|
"workflow_ref": expected_workflow_ref,
|
|
}
|
|
if any(manifest.get(key) != value for key, value in required.items()):
|
|
raise SystemExit("bundle manifest context mismatch")
|
|
if not re.fullmatch(r"[0-9a-f]{40}", str(manifest.get("git_commit") or "")):
|
|
raise SystemExit("bundle manifest commit is invalid")
|
|
files = manifest.get("files")
|
|
if not isinstance(files, dict) or sorted(files) != sorted(expected):
|
|
raise SystemExit("bundle manifest file allowlist mismatch")
|
|
payloads = {}
|
|
for relative in expected:
|
|
member = archive.getmember(relative)
|
|
data = archive.extractfile(member).read()
|
|
receipt = files.get(relative) or {}
|
|
if receipt.get("size") != len(data) or receipt.get("sha256") != hashlib.sha256(data).hexdigest():
|
|
raise SystemExit(f"bundle hash mismatch: {relative}")
|
|
payloads[relative] = data
|
|
dispatcher_sha = hashlib.sha256(dispatcher_path.read_bytes()).hexdigest()
|
|
if dispatcher_sha != files["scripts/gcp_iap_operator.sh"]["sha256"]:
|
|
raise SystemExit("installed dispatcher does not match the reviewed main bundle")
|
|
manifest_bytes = archive.extractfile(manifest_member).read()
|
|
|
|
for relative, data in sorted({**payloads, "bundle-manifest.json": manifest_bytes}.items()):
|
|
destination = (bundle_dir / relative).resolve()
|
|
if bundle_dir not in destination.parents:
|
|
raise SystemExit("bundle destination escaped the run directory")
|
|
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
os.chmod(destination.parent, 0o700)
|
|
with destination.open("xb") as handle:
|
|
handle.write(data)
|
|
os.chmod(destination, 0o600)
|
|
PY
|
|
}
|
|
|
|
receive_and_validate_bundle() {
|
|
local operation="$1"
|
|
local request_id="$2"
|
|
local target_db="$3"
|
|
local upload run_dir archive bundle_dir
|
|
|
|
require_root
|
|
upload="$(remote_upload_path "$request_id")"
|
|
run_dir="$REMOTE_RUN_ROOT/$request_id"
|
|
if [[ "$operation" == "direct-claim-replay" ]]; then
|
|
[[ ! -e "$run_dir" ]] || die "request_id already owns a remote run directory"
|
|
install -d -o root -g root -m 0700 "$run_dir"
|
|
write_run_marker "$run_dir/.run-owner" "$request_id" "$target_db"
|
|
else
|
|
validate_run_dir "$run_dir" "$request_id" "$target_db"
|
|
fi
|
|
archive="$run_dir/$operation-bundle.tar.gz"
|
|
bundle_dir="$run_dir/$operation-bundle"
|
|
[[ ! -e "$archive" && ! -e "$bundle_dir" ]] || die "operation bundle already exists in run directory"
|
|
mv -- "$upload" "$archive"
|
|
chown root:root "$archive"
|
|
chmod 0600 "$archive"
|
|
install -d -o root -g root -m 0700 "$bundle_dir"
|
|
validate_and_extract_bundle "$archive" "$bundle_dir" "$operation" "$request_id" "$target_db" || {
|
|
die "bundle membership, hash, context, or dispatcher validation failed"
|
|
}
|
|
VALIDATED_BUNDLE_DIR="$bundle_dir"
|
|
VALIDATED_BUNDLE_SHA256="$(sha256sum "$archive" | awk '{print $1}')"
|
|
VALIDATED_BUNDLE_COMMIT="$(
|
|
python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["git_commit"])' \
|
|
"$bundle_dir/bundle-manifest.json"
|
|
)"
|
|
}
|
|
|
|
remote_direct_claim_replay() {
|
|
local request_id="$1"
|
|
local target_db="$2"
|
|
local receipt_relative receipt_path run_dir output_path stdout_path stderr_path report_sha result_bytes
|
|
local caller_home caller_group export_path fixed_baseline_validation
|
|
|
|
receive_and_validate_bundle "direct-claim-replay" "$request_id" "$target_db"
|
|
receipt_relative="$(parity_receipt_relative "$target_db")"
|
|
receipt_path="$VALIDATED_BUNDLE_DIR/$receipt_relative"
|
|
[[ -s "$receipt_path" && ! -L "$receipt_path" ]] || die "target-specific tracked parity receipt is absent"
|
|
[[ -d "$LIVE_PROFILE" ]] || die "live profile required for temporary copy is absent"
|
|
|
|
run_dir="$REMOTE_RUN_ROOT/$request_id"
|
|
output_path="$run_dir/direct-claim-result.json"
|
|
stdout_path="$run_dir/direct-claim.stdout"
|
|
stderr_path="$run_dir/direct-claim.stderr"
|
|
: >"$stdout_path"
|
|
: >"$stderr_path"
|
|
chmod 0600 "$stdout_path" "$stderr_path"
|
|
|
|
if ! (
|
|
cd "$VALIDATED_BUNDLE_DIR"
|
|
/usr/bin/python3 scripts/run_gcp_generated_db_direct_claim_suite.py \
|
|
--execute \
|
|
--target-db "$target_db" \
|
|
--cloudsql-tool hermes-agent/leoclean-bin/cloudsql_memory_tool.py \
|
|
--manifest-sql ops/postgres_parity_manifest.sql \
|
|
--parity-receipt "$receipt_path" \
|
|
--output "$output_path" \
|
|
--host "$CLOUDSQL_HOST" \
|
|
--project "$PROJECT_ID" \
|
|
--password-secret "$CLOUDSQL_SECRET" \
|
|
--service "$LIVE_SERVICE" \
|
|
--live-profile "$LIVE_PROFILE" \
|
|
--run-id "$request_id"
|
|
) >"$stdout_path" 2>"$stderr_path"; then
|
|
die "reviewed six-turn clone-bound replay failed; private run files were retained for cleanup"
|
|
fi
|
|
|
|
[[ -s "$output_path" && ! -L "$output_path" ]] || die "six-turn replay did not retain its private receipt"
|
|
python3 - "$output_path" "$target_db" "$CORY_REPLAY_DB" "$CORY_REPLAY_FINGERPRINT" <<'PY'
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
payload = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
|
|
target_db = sys.argv[2]
|
|
expected_db = sys.argv[3]
|
|
expected_fingerprint = sys.argv[4]
|
|
checks = payload.get("checks") or {}
|
|
required = {
|
|
"status": payload.get("status") == "pass",
|
|
"target": payload.get("target_database") == target_db,
|
|
"six_replies": checks.get("six_replies_returned") is True,
|
|
"no_message_send": payload.get("posted_to_telegram") is False,
|
|
"service_unchanged": checks.get("live_service_unchanged") is True,
|
|
"profile_unchanged": checks.get("live_profile_unchanged") is True,
|
|
"database_unchanged": checks.get("generated_database_unchanged") is True,
|
|
"temporary_profile_removed": checks.get("temporary_profile_removed") is True,
|
|
}
|
|
if target_db == expected_db:
|
|
expected_counts = {
|
|
"claims": 1837,
|
|
"sources": 4145,
|
|
"claim_edges": 4916,
|
|
"claim_evidence": 4670,
|
|
"kb_proposals": 26,
|
|
}
|
|
before_fingerprint = payload.get("database_fingerprint_before") or {}
|
|
after_fingerprint = payload.get("database_fingerprint_after") or {}
|
|
before_counts = ((payload.get("canonical_status_before") or {}).get("high_signal_rows") or {})
|
|
after_counts = ((payload.get("canonical_status_after") or {}).get("high_signal_rows") or {})
|
|
required.update(
|
|
{
|
|
"fixed_fingerprint_before": before_fingerprint.get("sha256") == expected_fingerprint,
|
|
"fixed_fingerprint_after": after_fingerprint.get("sha256") == expected_fingerprint,
|
|
"fixed_counts_before": all(before_counts.get(key) == value for key, value in expected_counts.items()),
|
|
"fixed_counts_after": all(after_counts.get(key) == value for key, value in expected_counts.items()),
|
|
"strict_six_of_six": (payload.get("score") or {}).get("passes") == 6
|
|
and (payload.get("score") or {}).get("expected_prompt_count") == 6,
|
|
}
|
|
)
|
|
failed = sorted(name for name, ok in required.items() if not ok)
|
|
if failed:
|
|
raise SystemExit("replay receipt failed fixed checks: " + ", ".join(failed))
|
|
PY
|
|
chmod 0600 "$output_path"
|
|
report_sha="$(sha256sum "$output_path" | awk '{print $1}')"
|
|
result_bytes="$(wc -c <"$output_path" | tr -d ' ')"
|
|
caller_home="$(getent passwd "$SUDO_USER" | cut -d: -f6)"
|
|
caller_group="$(id -gn "$SUDO_USER")"
|
|
[[ "$caller_home" == /* && -d "$caller_home" && ! -L "$caller_home" ]] || {
|
|
die "OS Login caller home is unsafe for replay receipt export"
|
|
}
|
|
[[ "$(realpath "$caller_home")" == "$caller_home" ]] || die "OS Login caller home path drifted"
|
|
export_path="$caller_home/.teleo-iap-result-$request_id.json"
|
|
[[ ! -e "$export_path" ]] || die "request-derived replay receipt export already exists"
|
|
install -o "$SUDO_USER" -g "$caller_group" -m 0600 "$output_path" "$export_path"
|
|
[[ "$(sha256sum "$export_path" | awk '{print $1}')" == "$report_sha" ]] || {
|
|
die "exported replay receipt hash mismatch"
|
|
}
|
|
fixed_baseline_validation="not_applicable"
|
|
if [[ "$target_db" == "$CORY_REPLAY_DB" ]]; then
|
|
fixed_baseline_validation="pass"
|
|
fi
|
|
emit_json \
|
|
"direct-claim-replay" "$request_id" "$target_db" "pass" \
|
|
"bundle_commit=$VALIDATED_BUNDLE_COMMIT" \
|
|
"bundle_sha256=$VALIDATED_BUNDLE_SHA256" \
|
|
"result_sha256=$report_sha" \
|
|
"result_bytes=$result_bytes" \
|
|
"full_result_export_ready=true" \
|
|
"fixed_baseline_validation=$fixed_baseline_validation" \
|
|
"no_message_send=true" \
|
|
"live_service_unchanged=true" \
|
|
"live_profile_unchanged=true"
|
|
}
|
|
|
|
remote_cleanup_clone() {
|
|
local request_id="$1"
|
|
local target_db="$2"
|
|
local run_dir password exists remaining legacy_dir legacy_resolved
|
|
local caller_home export_path export_state
|
|
local rollback_exists rollback_allowconn rollback_connections leftover_temp_count temp_path
|
|
|
|
receive_and_validate_bundle "cleanup-clone" "$request_id" "$target_db"
|
|
run_dir="$REMOTE_RUN_ROOT/$request_id"
|
|
validate_run_dir "$run_dir" "$request_id" "$target_db"
|
|
legacy_dir=""
|
|
if [[ "$target_db" == "$CORY_REPLAY_DB" ]]; then
|
|
legacy_dir="$CORY_REPLAY_LEGACY_DIR"
|
|
if [[ -e "$legacy_dir" || -L "$legacy_dir" ]]; then
|
|
[[ -d "$legacy_dir" && ! -L "$legacy_dir" ]] || die "fixed legacy replay path is unsafe"
|
|
legacy_resolved="$(realpath "$legacy_dir")"
|
|
[[ "$legacy_resolved" == "$CORY_REPLAY_LEGACY_DIR" ]] || die "fixed legacy replay path drifted"
|
|
fi
|
|
fi
|
|
caller_home="$(getent passwd "$SUDO_USER" | cut -d: -f6)"
|
|
[[ "$caller_home" == /* && -d "$caller_home" && ! -L "$caller_home" ]] || {
|
|
die "OS Login caller home is unsafe for cleanup"
|
|
}
|
|
[[ "$(realpath "$caller_home")" == "$caller_home" ]] || die "OS Login caller home path drifted"
|
|
export_path="$caller_home/.teleo-iap-result-$request_id.json"
|
|
if [[ -e "$export_path" || -L "$export_path" ]]; then
|
|
[[ -f "$export_path" && ! -L "$export_path" ]] || die "request-derived replay export is unsafe"
|
|
export_state="$(stat -c '%U:%a' "$export_path")"
|
|
[[ "$export_state" == "$SUDO_USER:600" ]] || die "request-derived replay export ownership or mode drifted"
|
|
fi
|
|
while IFS= read -r -d '' temp_path; do
|
|
validate_replay_temp_root "$temp_path" "$request_id"
|
|
done < <(
|
|
find /tmp -mindepth 1 -maxdepth 1 \
|
|
-name "leo-clone-checkpoint-$request_id-gcp-direct-claim-*" -print0
|
|
)
|
|
command -v gcloud >/dev/null 2>&1 || die "gcloud is unavailable for secret-backed clone cleanup"
|
|
command -v psql >/dev/null 2>&1 || die "psql is unavailable for clone cleanup"
|
|
|
|
password="$(gcloud secrets versions access latest --secret="$CLOUDSQL_SECRET" --project="$PROJECT_ID" --quiet)"
|
|
[[ -n "$password" ]] || die "clone cleanup credential resolved empty"
|
|
export PGPASSWORD="$password"
|
|
export PGOPTIONS="-c statement_timeout=30000"
|
|
exists="$(
|
|
psql "host=$CLOUDSQL_HOST port=5432 dbname=postgres user=postgres sslmode=require connect_timeout=8" \
|
|
-X -Atq -v ON_ERROR_STOP=1 -v "target_db=$target_db" \
|
|
-c "select exists(select 1 from pg_database where datname = :'target_db');"
|
|
)"
|
|
[[ "$exists" == "t" || "$exists" == "f" ]] || die "clone existence readback was invalid"
|
|
rollback_exists="$(
|
|
psql "host=$CLOUDSQL_HOST port=5432 dbname=postgres user=postgres sslmode=require connect_timeout=8" \
|
|
-X -Atq -v ON_ERROR_STOP=1 -v "rollback_db=$DISABLED_ROLLBACK_DB" \
|
|
-c "select count(*) from pg_database where datname = :'rollback_db';"
|
|
)"
|
|
rollback_allowconn="$(
|
|
psql "host=$CLOUDSQL_HOST port=5432 dbname=postgres user=postgres sslmode=require connect_timeout=8" \
|
|
-X -Atq -v ON_ERROR_STOP=1 -v "rollback_db=$DISABLED_ROLLBACK_DB" \
|
|
-c "select datallowconn from pg_database where datname = :'rollback_db';"
|
|
)"
|
|
rollback_connections="$(
|
|
psql "host=$CLOUDSQL_HOST port=5432 dbname=postgres user=postgres sslmode=require connect_timeout=8" \
|
|
-X -Atq -v ON_ERROR_STOP=1 -v "rollback_db=$DISABLED_ROLLBACK_DB" \
|
|
-c "select count(*) from pg_stat_activity where datname = :'rollback_db';"
|
|
)"
|
|
[[ "$rollback_exists" == "1" ]] || die "disabled rollback database is absent"
|
|
[[ "$rollback_allowconn" == "f" ]] || die "rollback database unexpectedly allows connections"
|
|
[[ "$rollback_connections" == "0" ]] || die "rollback database has active connections"
|
|
|
|
if [[ -n "$legacy_dir" && ( -e "$legacy_dir" || -L "$legacy_dir" ) ]]; then
|
|
[[ -d "$legacy_dir" && ! -L "$legacy_dir" ]] || die "fixed legacy replay path became unsafe"
|
|
[[ "$(realpath "$legacy_dir")" == "$CORY_REPLAY_LEGACY_DIR" ]] || {
|
|
die "fixed legacy replay path changed before cleanup"
|
|
}
|
|
rm -rf --one-file-system -- "$legacy_dir"
|
|
fi
|
|
[[ -z "$legacy_dir" || ( ! -e "$legacy_dir" && ! -L "$legacy_dir" ) ]] || {
|
|
die "fixed legacy replay directory remained"
|
|
}
|
|
if [[ -e "$export_path" || -L "$export_path" ]]; then
|
|
[[ -f "$export_path" && ! -L "$export_path" ]] || die "request-derived replay export became unsafe"
|
|
rm -f -- "$export_path"
|
|
fi
|
|
[[ ! -e "$export_path" && ! -L "$export_path" ]] || die "request-derived replay export remained"
|
|
while IFS= read -r -d '' temp_path; do
|
|
validate_replay_temp_root "$temp_path" "$request_id"
|
|
rm -rf --one-file-system -- "$temp_path"
|
|
done < <(
|
|
find /tmp -mindepth 1 -maxdepth 1 \
|
|
-name "leo-clone-checkpoint-$request_id-gcp-direct-claim-*" -print0
|
|
)
|
|
leftover_temp_count="$(
|
|
find /tmp -mindepth 1 -maxdepth 1 \
|
|
-name "leo-clone-checkpoint-$request_id-gcp-direct-claim-*" -print | wc -l | tr -d ' '
|
|
)"
|
|
[[ "$leftover_temp_count" == "0" ]] || die "request-derived temporary replay paths remained"
|
|
|
|
if [[ "$exists" == "t" ]]; then
|
|
psql "host=$CLOUDSQL_HOST port=5432 dbname=postgres user=postgres sslmode=require connect_timeout=8" \
|
|
-X -q -v ON_ERROR_STOP=1 -v "target_db=$target_db" <<'SQL'
|
|
select pg_terminate_backend(pid)
|
|
from pg_stat_activity
|
|
where datname = :'target_db' and pid <> pg_backend_pid();
|
|
drop database :"target_db";
|
|
SQL
|
|
fi
|
|
remaining="$(
|
|
psql "host=$CLOUDSQL_HOST port=5432 dbname=postgres user=postgres sslmode=require connect_timeout=8" \
|
|
-X -Atq -v ON_ERROR_STOP=1 -v "target_db=$target_db" \
|
|
-c "select count(*) from pg_database where datname = :'target_db';"
|
|
)"
|
|
unset PGPASSWORD PGOPTIONS
|
|
password=""
|
|
[[ "$remaining" == "0" ]] || die "clone database remained after cleanup"
|
|
|
|
validate_run_dir "$run_dir" "$request_id" "$target_db"
|
|
rm -rf --one-file-system -- "$run_dir"
|
|
[[ ! -e "$run_dir" && ! -L "$run_dir" ]] || die "exact run-owned directory remained after cleanup"
|
|
emit_json \
|
|
"cleanup-clone" "$request_id" "$target_db" "pass" \
|
|
"bundle_commit=$VALIDATED_BUNDLE_COMMIT" \
|
|
"bundle_sha256=$VALIDATED_BUNDLE_SHA256" \
|
|
"clone_database_remaining=0" \
|
|
"run_directory_remaining=0" \
|
|
"legacy_directory_remaining=0" \
|
|
"export_file_remaining=0" \
|
|
"leftover_temp_directory_count=$leftover_temp_count" \
|
|
"rollback_database_datallowconn=false" \
|
|
"rollback_database_connections=$rollback_connections"
|
|
}
|
|
|
|
remote_main() {
|
|
[[ "$#" -eq 3 ]] || die "remote mode requires operation, request_id, and target_db"
|
|
local operation="$1"
|
|
local request_id="$2"
|
|
local target_db="$3"
|
|
validate_inputs "$operation" "$request_id" "$target_db"
|
|
case "$operation" in
|
|
status) remote_status "$request_id" "$target_db" ;;
|
|
direct-claim-replay) remote_direct_claim_replay "$request_id" "$target_db" ;;
|
|
cleanup-clone) remote_cleanup_clone "$request_id" "$target_db" ;;
|
|
esac
|
|
}
|
|
|
|
write_sanitized_result() {
|
|
local output_path="$1"
|
|
local raw_stdout="$2"
|
|
local operation="$3"
|
|
local request_id="$4"
|
|
local target_db="$5"
|
|
local exit_code="$6"
|
|
python3 "$RESULT_HELPER" sanitize \
|
|
--result-path "$output_path" \
|
|
--raw-stdout "$raw_stdout" \
|
|
--operation "$operation" \
|
|
--request-id "$request_id" \
|
|
--target-db "$target_db" \
|
|
--exit-code "$exit_code"
|
|
chmod 0600 "$output_path"
|
|
}
|
|
|
|
validate_local_bundle() {
|
|
local request_id="$1"
|
|
local expected path resolved bundle_root mode
|
|
|
|
bundle_root="$RUNNER_TEMP/gcp-iap-operator-bundle"
|
|
expected="$bundle_root/$request_id.tar.gz"
|
|
path="$expected"
|
|
[[ -f "$path" && ! -L "$path" ]] || die "fixed workflow bundle is absent or unsafe"
|
|
resolved="$(realpath "$path")"
|
|
[[ "$resolved" == "$expected" ]] || die "workflow bundle escaped its request-derived path"
|
|
mode="$(python3 -c 'import os,stat,sys; print(oct(stat.S_IMODE(os.stat(sys.argv[1]).st_mode))[2:])' "$path")"
|
|
[[ "$mode" == "600" ]] || die "workflow bundle mode is not private"
|
|
printf '%s\n' "$path"
|
|
}
|
|
|
|
validate_downloaded_replay() {
|
|
local result_path="$1"
|
|
local full_result="$2"
|
|
local target_db="$3"
|
|
python3 "$RESULT_HELPER" validate-replay \
|
|
--result-path "$result_path" \
|
|
--full-result "$full_result" \
|
|
--target-db "$target_db" \
|
|
--expected-cory-replay-db "$CORY_REPLAY_DB" \
|
|
--expected-cory-replay-fingerprint "$CORY_REPLAY_FINGERPRINT"
|
|
}
|
|
|
|
mark_result_export_failure() {
|
|
local result_path="$1"
|
|
local exit_code="$2"
|
|
local cleanup_exit_code="$3"
|
|
python3 "$RESULT_HELPER" mark-export-failure \
|
|
--result-path "$result_path" \
|
|
--exit-code "$exit_code" \
|
|
--cleanup-exit-code "$cleanup_exit_code"
|
|
}
|
|
|
|
runner_main() {
|
|
[[ "$#" -eq 3 ]] || die "runner mode requires operation, request_id, and target_db"
|
|
local operation="$1"
|
|
local request_id="$2"
|
|
local target_db="$3"
|
|
local temp_dir result_dir result_path raw_stdout raw_stderr scp_stdout ssh_key remote_command exit_code
|
|
local sanitized_status bundle_path remote_upload full_result remote_result_export export_cleanup_command
|
|
local replay_exit_code export_cleanup_exit_code
|
|
local -a remote_argv
|
|
|
|
validate_inputs "$operation" "$request_id" "$target_db"
|
|
[[ -n "${RUNNER_TEMP:-}" && "$RUNNER_TEMP" == /* ]] || die "RUNNER_TEMP must be an absolute runner path"
|
|
[[ -n "${RESULT_DIR:-}" && "$RESULT_DIR" == "$RUNNER_TEMP"/* ]] || {
|
|
die "RESULT_DIR must be a fixed child of RUNNER_TEMP"
|
|
}
|
|
command -v gcloud >/dev/null 2>&1 || die "gcloud is unavailable"
|
|
install -d -m 0700 "$RESULT_DIR"
|
|
result_dir="$(realpath "$RESULT_DIR")"
|
|
[[ "$result_dir" == "$RUNNER_TEMP"/* ]] || die "RESULT_DIR escaped RUNNER_TEMP"
|
|
result_path="$result_dir/result.json"
|
|
: >"$result_path"
|
|
chmod 0600 "$result_path"
|
|
|
|
temp_dir="$(mktemp -d "$RUNNER_TEMP/teleo-iap-operator.XXXXXX")"
|
|
RUNNER_TEMP_DIR="$temp_dir"
|
|
chmod 0700 "$temp_dir"
|
|
trap cleanup_runner_temp EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
raw_stdout="$temp_dir/remote.stdout"
|
|
raw_stderr="$temp_dir/transport.stderr"
|
|
scp_stdout="$temp_dir/scp.stdout"
|
|
ssh_key="$temp_dir/ssh_key"
|
|
: >"$raw_stdout"
|
|
: >"$raw_stderr"
|
|
: >"$scp_stdout"
|
|
chmod 0600 "$raw_stdout" "$raw_stderr" "$scp_stdout"
|
|
|
|
if [[ "$operation" != "status" ]]; then
|
|
bundle_path="$(validate_local_bundle "$request_id")"
|
|
remote_upload="$INSTANCE:.teleo-iap-upload-$request_id.tar.gz"
|
|
set +e
|
|
gcloud compute scp "$bundle_path" "$remote_upload" \
|
|
--project="$PROJECT_ID" \
|
|
--zone="$ZONE" \
|
|
--tunnel-through-iap \
|
|
--ssh-key-file="$ssh_key" \
|
|
--ssh-key-expire-after=5m \
|
|
--scp-flag=-p \
|
|
--quiet >"$scp_stdout" 2>>"$raw_stderr"
|
|
exit_code=$?
|
|
set -e
|
|
if [[ "$exit_code" -ne 0 ]]; then
|
|
write_sanitized_result "$result_path" "$raw_stdout" "$operation" "$request_id" "$target_db" "$exit_code"
|
|
printf 'sanitized_result=%s\n' "$result_path"
|
|
return "$exit_code"
|
|
fi
|
|
fi
|
|
|
|
if [[ "$operation" == "status" ]]; then
|
|
remote_argv=("$REMOTE_DISPATCHER" --remote "$operation" "$request_id" "$target_db")
|
|
else
|
|
remote_argv=(sudo -n "$REMOTE_DISPATCHER" --remote "$operation" "$request_id" "$target_db")
|
|
fi
|
|
printf -v remote_command '%q ' "${remote_argv[@]}"
|
|
|
|
set +e
|
|
gcloud compute ssh "$INSTANCE" \
|
|
--project="$PROJECT_ID" \
|
|
--zone="$ZONE" \
|
|
--tunnel-through-iap \
|
|
--ssh-key-file="$ssh_key" \
|
|
--ssh-key-expire-after=5m \
|
|
--command="$remote_command" \
|
|
--quiet >"$raw_stdout" 2>>"$raw_stderr"
|
|
exit_code=$?
|
|
set -e
|
|
[[ ! -e "$ssh_key" ]] || chmod 0600 "$ssh_key"
|
|
[[ ! -e "$ssh_key.pub" ]] || chmod 0600 "$ssh_key.pub"
|
|
write_sanitized_result "$result_path" "$raw_stdout" "$operation" "$request_id" "$target_db" "$exit_code"
|
|
sanitized_status="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["status"])' "$result_path")"
|
|
if [[ "$exit_code" -eq 0 && "$sanitized_status" != "pass" ]]; then
|
|
exit_code=90
|
|
fi
|
|
if [[ "$exit_code" -eq 0 && "$operation" == "direct-claim-replay" ]]; then
|
|
full_result="$result_dir/direct-claim-result.json"
|
|
[[ ! -e "$full_result" && ! -L "$full_result" ]] || die "local full replay result path was not empty"
|
|
remote_result_export="$INSTANCE:.teleo-iap-result-$request_id.json"
|
|
set +e
|
|
gcloud compute scp "$remote_result_export" "$full_result" \
|
|
--project="$PROJECT_ID" \
|
|
--zone="$ZONE" \
|
|
--tunnel-through-iap \
|
|
--ssh-key-file="$ssh_key" \
|
|
--ssh-key-expire-after=5m \
|
|
--quiet >>"$scp_stdout" 2>>"$raw_stderr"
|
|
exit_code=$?
|
|
set -e
|
|
if [[ "$exit_code" -eq 0 ]]; then
|
|
chmod 0600 "$full_result"
|
|
set +e
|
|
validate_downloaded_replay "$result_path" "$full_result" "$target_db"
|
|
exit_code=$?
|
|
set -e
|
|
fi
|
|
rm -f -- "$full_result"
|
|
replay_exit_code="$exit_code"
|
|
printf -v export_cleanup_command '%q ' rm -f -- ".teleo-iap-result-$request_id.json"
|
|
set +e
|
|
gcloud compute ssh "$INSTANCE" \
|
|
--project="$PROJECT_ID" \
|
|
--zone="$ZONE" \
|
|
--tunnel-through-iap \
|
|
--ssh-key-file="$ssh_key" \
|
|
--ssh-key-expire-after=5m \
|
|
--command="$export_cleanup_command" \
|
|
--quiet >>"$scp_stdout" 2>>"$raw_stderr"
|
|
export_cleanup_exit_code=$?
|
|
set -e
|
|
if [[ "$replay_exit_code" -ne 0 ]]; then
|
|
exit_code="$replay_exit_code"
|
|
else
|
|
exit_code="$export_cleanup_exit_code"
|
|
fi
|
|
if [[ "$exit_code" -ne 0 ]]; then
|
|
mark_result_export_failure "$result_path" "$exit_code" "$export_cleanup_exit_code"
|
|
else
|
|
python3 "$RESULT_HELPER" mark-export-removed --result-path "$result_path"
|
|
fi
|
|
fi
|
|
printf 'sanitized_result=%s\n' "$result_path"
|
|
return "$exit_code"
|
|
}
|
|
|
|
if [[ "${1:-}" == "--remote" ]]; then
|
|
shift
|
|
remote_main "$@"
|
|
else
|
|
runner_main "$@"
|
|
fi
|