teleo-infrastructure/deploy/sync-gcp-leoclean-runtime.sh

#!/usr/bin/env bash
# Deploy the source-controlled Cloud SQL bridge and non-secret service settings
# to the GCP parallel leoclean runtime. This does not create database users,
# read secret values, send Telegram messages, or promote Cloud SQL.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
PROJECT="${PROJECT:-teleo-501523}"
ZONE="${ZONE:-europe-west6-a}"
INSTANCE="${INSTANCE:-teleo-prod-1}"
SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}"
MERGED_REF="${MERGED_REF:-origin/main}"
REMOTE_PROFILE_BIN="${REMOTE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}"
REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}"
REMOTE_CLOUDSDK_CONFIG="${REMOTE_CLOUDSDK_CONFIG:-/run/leoclean-gcp-prod-parallel}"
EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}"

WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb"
TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py"
DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf"
DROPIN_NAME="10-cloudsql-memory.conf"

DRY_RUN=false
RESTART=false
VERIFY_ONLY=false

usage() {
  cat <<'USAGE'
Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--restart] [--verify-only]

The database role and scoped Secret Manager secret must exist before --restart.
This script deploys only the bridge files and non-secret systemd drop-in.
USAGE
}

for arg in "$@"; do
  case "$arg" in
    --dry-run) DRY_RUN=true ;;
    --restart) RESTART=true ;;
    --verify-only) VERIFY_ONLY=true ;;
    --help|-h)
      usage
      exit 0
      ;;
    *)
      echo "Unknown arg: $arg" >&2
      usage >&2
      exit 64
      ;;
  esac
done

for path in "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; do
  if [ ! -f "$REPO_ROOT/$path" ]; then
    echo "Required source file is missing: $path" >&2
    exit 66
  fi
done

if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then
  echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2
  exit 65
fi
if ! grep -Fxq "Environment=CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG" "$REPO_ROOT/$DROPIN_REL"; then
  echo "Runtime drop-in CLOUDSDK_CONFIG does not match $REMOTE_CLOUDSDK_CONFIG." >&2
  exit 65
fi

SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)"
if ! git -C "$REPO_ROOT" diff --quiet HEAD -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; then
  echo "Refusing to deploy runtime files that do not match HEAD." >&2
  exit 65
fi
if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL")" ]; then
  echo "Refusing to deploy untracked runtime files." >&2
  exit 65
fi
if ! git -C "$REPO_ROOT" rev-parse --verify "$MERGED_REF^{commit}" >/dev/null 2>&1; then
  echo "Merged reference is unavailable: $MERGED_REF" >&2
  exit 69
fi
if ! git -C "$REPO_ROOT" merge-base --is-ancestor "$SOURCE_REVISION" "$MERGED_REF"; then
  echo "Refusing to deploy unmerged revision $SOURCE_REVISION (not contained in $MERGED_REF)." >&2
  exit 65
fi
WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')"
TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')"
DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')"

GCP_SSH=(
  gcloud compute ssh "$INSTANCE"
  --project="$PROJECT"
  --zone="$ZONE"
  --tunnel-through-iap
)

remote_helpers=$(cat <<'REMOTE'
assert_service_running() {
  local active_state sub_state main_pid unit_user process_user
  active_state="$(systemctl show "$SERVICE" --property=ActiveState --value)"
  sub_state="$(systemctl show "$SERVICE" --property=SubState --value)"
  if [ "$active_state" != "active" ] || [ "$sub_state" != "running" ]; then
    echo "Service is not active/running: ActiveState=$active_state SubState=$sub_state" >&2
    return 1
  fi

  main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
  unit_user="$(systemctl show "$SERVICE" --property=User --value)"
  case "$main_pid" in
    ''|0|*[!0-9]*)
      echo "Service has no live MainPID: $main_pid" >&2
      return 1
      ;;
  esac
  if [ "$unit_user" != "teleo" ]; then
    echo "Service unit user is not teleo: $unit_user" >&2
    return 1
  fi
  process_user="$(ps -o user= -p "$main_pid" | awk '{print $1}')"
  if [ "$process_user" != "teleo" ]; then
    echo "Service process user is not teleo: $process_user" >&2
    return 1
  fi
}

assert_runtime_environment() {
  local environment runtime_dir_state
  environment="$(systemctl show "$SERVICE" --property=Environment --value)"
  case " $environment " in
    *" PGPASSWORD="*)
      echo "Service environment still contains PGPASSWORD." >&2
      return 1
      ;;
  esac
  case " $environment " in
    *" CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG "*) ;;
    *)
      echo "Service does not use the isolated CLOUDSDK_CONFIG." >&2
      return 1
      ;;
  esac
  runtime_dir_state="$(stat -c '%U:%G:%a' "$REMOTE_CLOUDSDK_CONFIG")"
  if [ "$runtime_dir_state" != "teleo:teleo:700" ]; then
    echo "Isolated CLOUDSDK_CONFIG has unexpected ownership or mode: $runtime_dir_state" >&2
    return 1
  fi
}

assert_attached_service_account() {
  local sdk_config="$1" metadata_email
  sudo install -d -o teleo -g teleo -m 0700 "$sdk_config"
  metadata_email="$(
    sudo -n -u teleo env -u PGPASSWORD \
      HOME=/home/teleo \
      CLOUDSDK_CONFIG="$sdk_config" \
      curl --fail --silent --show-error \
        --header 'Metadata-Flavor: Google' \
        'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email'
  )"
  if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then
    echo "Attached VM service account mismatch: $metadata_email" >&2
    return 1
  fi

  # An empty, phase-local Cloud SDK config forces this token request to use
  # the VM's attached identity instead of an operator's cached credentials.
  sudo -n -u teleo env -u PGPASSWORD \
    HOME=/home/teleo \
    CLOUDSDK_CONFIG="$sdk_config" \
    CLOUDSDK_CORE_PROJECT="$PROJECT" \
    gcloud --quiet auth print-access-token >/dev/null
  echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email"
}

run_scoped_status() {
  local phase="$1" tool_path="$2" sdk_config="$3"
  assert_attached_service_account "$sdk_config"
  echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical"
  sudo -n -u teleo env -u PGPASSWORD \
    HOME=/home/teleo \
    CLOUDSDK_CONFIG="$sdk_config" \
    CLOUDSDK_CORE_PROJECT="$PROJECT" \
    TELEO_GCP_PROJECT=teleo-501523 \
    TELEO_CLOUDSQL_HOST=10.61.0.3 \
    TELEO_CLOUDSQL_PORT=5432 \
    TELEO_CLOUDSQL_DB=teleo_canonical \
    TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical \
    TELEO_CLOUDSQL_USER=leoclean_kb_runtime \
    TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \
    TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \
    python3 "$tool_path" status --format json --redacted
}

assert_deployed_files() {
  local wrapper tool dropin
  wrapper="$REMOTE_PROFILE_BIN/teleo-kb"
  tool="$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py"
  dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME"
  test "$(sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA"
  test "$(sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA"
  test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA"
  sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin"
  sudo grep -E '^Environment=(CLOUDSDK_CONFIG|TELEO_(KB_MODE|GCP_PROJECT|CLOUDSQL_|CANONICAL_CLOUDSQL_DB))' "$dropin"
}
REMOTE
)

printf -v verify_context \
  'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\n' \
  "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \
  "$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \
  "$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA"

verify_body=$(cat <<'REMOTE'
set -euo pipefail
verification_tmp="$(mktemp -d /tmp/teleo-gcp-runtime-verify.XXXXXX)"
cleanup_verification() { sudo rm -rf -- "$verification_tmp"; }
trap cleanup_verification EXIT
assert_deployed_files
assert_service_running
assert_runtime_environment
run_scoped_status verify "$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py" "$verification_tmp/gcloud"
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
REMOTE
)
verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}"

if $DRY_RUN; then
  echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE restart=$RESTART"
  echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA dropin_sha256=$DROPIN_SHA"
  exit 0
fi

if $VERIFY_ONLY; then
  "${GCP_SSH[@]}" --command="$verify_command"
  exit 0
fi

if ! $RESTART; then
  echo "Refusing to install runtime files without --restart; use --dry-run or --verify-only for read-only work." >&2
  exit 64
fi

REMOTE_TMP="/tmp/teleo-gcp-leoclean-runtime-$(date -u +%Y%m%dT%H%M%SZ)-$$"
printf -v sync_context \
  'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\nREMOTE_TMP=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\n' \
  "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \
  "$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \
  "$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA" "$REMOTE_TMP" \
  "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" "$SOURCE_REVISION"

sync_body=$(cat <<'REMOTE'
set -euo pipefail
remote_tmp="$REMOTE_TMP"
profile_bin="$REMOTE_PROFILE_BIN"
dropin_dir="$REMOTE_DROPIN_DIR"
backup_dir="$remote_tmp/backup"
dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$"
mutation_started=false

backup_path() {
  local target="$1" name="$2"
  if sudo test -e "$target"; then
    sudo cp -a -- "$target" "$backup_dir/$name"
    touch "$backup_dir/$name.present"
  fi
}

restore_path() {
  local target="$1" name="$2" restore_rc=0 restore_tmp
  if [ -f "$backup_dir/$name.present" ]; then
    restore_tmp="${target}.livingip-rollback.$$"
    sudo rm -f -- "$restore_tmp" || restore_rc=1
    if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then
      sudo mv -f -- "$restore_tmp" "$target" || restore_rc=1
    else
      restore_rc=1
    fi
    sudo rm -f -- "$restore_tmp" || restore_rc=1
  else
    sudo rm -f -- "$target" || restore_rc=1
  fi
  return "$restore_rc"
}

rollback_runtime() {
  local rollback_rc=0
  echo "Deployment failed; restoring the complete previous runtime set." >&2
  sudo systemctl stop "$SERVICE" || rollback_rc=1
  restore_path "$profile_bin/teleo-kb" wrapper || rollback_rc=1
  restore_path "$profile_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
  restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1
  restore_path "$profile_bin/.livingip-runtime-revision" revision || rollback_rc=1
  sudo systemctl daemon-reload || rollback_rc=1
  if [ "$rollback_rc" -eq 0 ]; then
    sudo systemctl start "$SERVICE" || rollback_rc=1
    assert_service_running || rollback_rc=1
  fi
  return "$rollback_rc"
}

on_exit() {
  local rc="$1" rollback_rc=0
  trap - EXIT INT TERM
  if [ "$rc" -ne 0 ] && [ "$mutation_started" = true ]; then
    set +e
    rollback_runtime
    rollback_rc=$?
    set -e
    if [ "$rollback_rc" -ne 0 ]; then
      echo "CRITICAL: previous runtime restoration did not return the service to active/running." >&2
      rc=1
    fi
  fi
  sudo rm -f -- "$dropin_stage" || true
  sudo rm -rf -- "$remote_tmp" || true
  exit "$rc"
}

trap 'on_exit $?' EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

sudo rm -rf -- "$remote_tmp"
mkdir -p "$backup_dir"
tar -C "$remote_tmp" -xf -

assert_service_running
run_scoped_status pre "$remote_tmp/$TOOL_REL" "$remote_tmp/gcloud-pre"

backup_path "$profile_bin/teleo-kb" wrapper
backup_path "$profile_bin/cloudsql_memory_tool.py" tool
backup_path "$dropin_dir/$DROPIN_NAME" dropin
backup_path "$profile_bin/.livingip-runtime-revision" revision

mutation_started=true
sudo systemctl stop "$SERVICE"
sudo install -d -o teleo -g teleo -m 0755 "$profile_bin"
sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$WRAPPER_REL" "$profile_bin/teleo-kb"
sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$TOOL_REL" "$profile_bin/cloudsql_memory_tool.py"
sudo install -d -o root -g root -m 0755 "$dropin_dir"
sudo install -o root -g root -m 0644 "$remote_tmp/$DROPIN_REL" "$dropin_stage"
sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME"
printf '%s\n' "$SOURCE_REVISION" | sudo tee "$profile_bin/.livingip-runtime-revision" >/dev/null
sudo chown teleo:teleo "$profile_bin/.livingip-runtime-revision"
sudo chmod 0644 "$profile_bin/.livingip-runtime-revision"
sudo systemctl daemon-reload
sudo systemctl start "$SERVICE"

assert_deployed_files
assert_service_running
assert_runtime_environment
run_scoped_status post "$profile_bin/cloudsql_memory_tool.py" "$remote_tmp/gcloud-post"
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
mutation_started=false
REMOTE
)
sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}"

COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \
  "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command"