#!/usr/bin/env bash
# Deploy the source-controlled Cloud SQL bridge and non-secret service settings
# to the GCP parallel leoclean runtime. This does not create database users,
# read secret values, send Telegram messages, or promote Cloud SQL.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
PROJECT="${PROJECT:-teleo-501523}"
ZONE="${ZONE:-europe-west6-a}"
INSTANCE="${INSTANCE:-teleo-prod-1}"
SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}"
MERGED_REF="${MERGED_REF:-origin/main}"
REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"
SERVICE_PROFILE_BIN="${SERVICE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}"
SERVICE_PROFILE_ROOT="${SERVICE_PROFILE_ROOT:-/home/teleo/.hermes/profiles/leoclean}"
SERVICE_HERMES_ROOT="${SERVICE_HERMES_ROOT:-/home/teleo/.hermes}"
REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}"
EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}"

WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb"
TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py"
PERMISSION_VERIFIER_REL="ops/verify_gcp_leoclean_runtime_permissions.py"
SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py"
DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf"
DROPIN_NAME="10-cloudsql-memory.conf"

DRY_RUN=false
RESTART=false
VERIFY_ONLY=false
PREFLIGHT_ONLY=false

usage() {
  cat <<'USAGE'
Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--preflight-only] [--restart] [--verify-only]

The database role and scoped Secret Manager secret must exist before --restart.
This script deploys only the bridge files and non-secret systemd drop-in.
USAGE
}

for arg in "$@"; do
  case "$arg" in
    --dry-run) DRY_RUN=true ;;
    --preflight-only) PREFLIGHT_ONLY=true ;;
    --restart) RESTART=true ;;
    --verify-only) VERIFY_ONLY=true ;;
    --help|-h)
      usage
      exit 0
      ;;
    *)
      echo "Unknown arg: $arg" >&2
      usage >&2
      exit 64
      ;;
  esac
done

if $PREFLIGHT_ONLY && $RESTART; then
  echo "--preflight-only and --restart are mutually exclusive." >&2
  exit 64
fi
if $VERIFY_ONLY && { $PREFLIGHT_ONLY || $RESTART; }; then
  echo "--verify-only cannot be combined with --preflight-only or --restart." >&2
  exit 64
fi

for path in \
  "$WRAPPER_REL" \
  "$TOOL_REL" \
  "$PERMISSION_VERIFIER_REL" \
  "$SERVICE_ENV_VERIFIER_REL" \
  "$DROPIN_REL"; do
  if [ ! -f "$REPO_ROOT/$path" ]; then
    echo "Required source file is missing: $path" >&2
    exit 66
  fi
  if [ -L "$REPO_ROOT/$path" ]; then
    echo "Refusing symlinked runtime source file: $path" >&2
    exit 65
  fi
done

if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then
  echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2
  exit 65
fi
if ! grep -Fxq 'Environment=TELEO_GCP_METADATA_ONLY=1' "$REPO_ROOT/$DROPIN_REL"; then
  echo "Runtime drop-in must enforce metadata-only GCP credentials." >&2
  exit 65
fi
if ! grep -Fxq "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$REPO_ROOT/$DROPIN_REL"; then
  echo "Runtime drop-in does not bind the root-controlled runtime into the service profile." >&2
  exit 65
fi
if ! grep -Fxq "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$REPO_ROOT/$DROPIN_REL"; then
  echo "Runtime drop-in does not protect the service-profile ancestor from replacement." >&2
  exit 65
fi
if ! grep -Fxq "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$REPO_ROOT/$DROPIN_REL"; then
  echo "Runtime drop-in does not limit service-profile writes to state and workspace." >&2
  exit 65
fi
if ! grep -Fxq 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$REPO_ROOT/$DROPIN_REL"; then
  echo "Runtime drop-in must remove CAP_SYS_ADMIN from the service." >&2
  exit 65
fi

SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)"
if ! git -C "$REPO_ROOT" diff --quiet HEAD -- \
  "$WRAPPER_REL" \
  "$TOOL_REL" \
  "$PERMISSION_VERIFIER_REL" \
  "$SERVICE_ENV_VERIFIER_REL" \
  "$DROPIN_REL"; then
  echo "Refusing to deploy runtime files that do not match HEAD." >&2
  exit 65
fi
if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- \
  "$WRAPPER_REL" \
  "$TOOL_REL" \
  "$PERMISSION_VERIFIER_REL" \
  "$SERVICE_ENV_VERIFIER_REL" \
  "$DROPIN_REL")" ]; then
  echo "Refusing to deploy untracked runtime files." >&2
  exit 65
fi
if ! git -C "$REPO_ROOT" rev-parse --verify "$MERGED_REF^{commit}" >/dev/null 2>&1; then
  echo "Merged reference is unavailable: $MERGED_REF" >&2
  exit 69
fi
if ! git -C "$REPO_ROOT" merge-base --is-ancestor "$SOURCE_REVISION" "$MERGED_REF"; then
  echo "Refusing to deploy unmerged revision $SOURCE_REVISION (not contained in $MERGED_REF)." >&2
  exit 65
fi
WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')"
TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')"
PERMISSION_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$PERMISSION_VERIFIER_REL" | awk '{print $1}')"
SERVICE_ENV_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')"
DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')"

GCP_SSH=(
  gcloud compute ssh "$INSTANCE"
  --project="$PROJECT"
  --zone="$ZONE"
  --tunnel-through-iap
)

remote_helpers=$(cat <<'REMOTE'
assert_service_running() {
  local active_state sub_state main_pid unit_user process_user
  active_state="$(systemctl show "$SERVICE" --property=ActiveState --value)"
  sub_state="$(systemctl show "$SERVICE" --property=SubState --value)"
  if [ "$active_state" != "active" ] || [ "$sub_state" != "running" ]; then
    echo "Service is not active/running: ActiveState=$active_state SubState=$sub_state" >&2
    return 1
  fi

  main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
  unit_user="$(systemctl show "$SERVICE" --property=User --value)"
  case "$main_pid" in
    ''|0|*[!0-9]*)
      echo "Service has no live MainPID: $main_pid" >&2
      return 1
      ;;
  esac
  if [ "$unit_user" != "teleo" ]; then
    echo "Service unit user is not teleo: $unit_user" >&2
    return 1
  fi
  process_user="$(ps -o user= -p "$main_pid" | awk '{print $1}')"
  if [ "$process_user" != "teleo" ]; then
    echo "Service process user is not teleo: $process_user" >&2
    return 1
  fi
}

assert_unit_configuration() {
  local require_reviewed="${1:-true}"
  local dropin_path dropin_name dropin_paths environment_files reviewed_dropin_seen=false
  dropin_paths="$(systemctl show "$SERVICE" --property=DropInPaths --value)"
  environment_files="$(systemctl show "$SERVICE" --property=EnvironmentFiles --value)"
  if [ -n "$environment_files" ]; then
    echo "Service has an effective EnvironmentFile, which is not allowed for the scoped runtime." >&2
    return 1
  fi
  for dropin_path in $dropin_paths; do
    if [ "$dropin_path" = "$REMOTE_DROPIN_DIR/$DROPIN_NAME" ]; then
      reviewed_dropin_seen=true
      continue
    fi
    dropin_name="${dropin_path##*/}"
    if [[ "$dropin_name" > "$DROPIN_NAME" ]]; then
      echo "Service has a later drop-in than the reviewed Cloud SQL configuration: $dropin_name" >&2
      return 1
    fi
  done
  if [ "$require_reviewed" = true ] && [ "$reviewed_dropin_seen" != true ]; then
    echo "Service does not load the reviewed Cloud SQL drop-in." >&2
    return 1
  fi
}

assert_runtime_environment() {
  local verifier_path="$1" require_reviewed="${2:-true}" main_pid
  main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
  sudo /usr/bin/python3 "$verifier_path" --pid "$main_pid"
  assert_unit_configuration "$require_reviewed"
}

assert_attached_service_account() {
  local metadata_email
  metadata_email="$(
    sudo -n -u teleo env -i \
      HOME=/home/teleo \
      PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
      /usr/bin/curl -q --noproxy '*' --proto '=http' --max-time 10 --fail --silent --show-error \
        --header 'Metadata-Flavor: Google' \
        'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email'
  )"
  if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then
    echo "Attached VM service account mismatch: $metadata_email" >&2
    return 1
  fi
  echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email"
}

assert_administrator_secret_denied() {
  local main_pid
  main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
  sudo nsenter -t "$main_pid" -m --env -- \
    /usr/bin/setpriv \
      --reuid=teleo \
      --regid=teleo \
      --init-groups \
      --no-new-privs \
      /usr/bin/python3 - "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" <<'PY'
import json
import sys
import urllib.error
import urllib.parse
import urllib.request

project, expected_email = sys.argv[1:]
metadata_headers = {"Metadata-Flavor": "Google"}
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))


def metadata(path: str) -> bytes:
    request = urllib.request.Request(
        "http://169.254.169.254/computeMetadata/v1/" + path,
        headers=metadata_headers,
    )
    with opener.open(request, timeout=10) as response:
        return response.read(65536)


try:
    email = metadata("instance/service-accounts/default/email").decode("utf-8").strip()
    if email != expected_email:
        raise RuntimeError
    token_payload = json.loads(metadata("instance/service-accounts/default/token"))
    token = token_payload.get("access_token")
    if token_payload.get("token_type") != "Bearer" or not isinstance(token, str) or not token:
        raise RuntimeError
    secret = urllib.parse.quote("gcp-teleo-pgvector-standby-postgres-password", safe="")
    url = f"https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{secret}/versions/latest:access"
    request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
    try:
        response = opener.open(request, timeout=10)
    except urllib.error.HTTPError as error:
        body = error.read(65536)
        parsed = json.loads(body)
        detail = parsed.get("error", {})
        message = str(detail.get("message", "")).lower()
        if error.code != 403 or detail.get("status") != "PERMISSION_DENIED":
            raise RuntimeError from None
        if "secretmanager.versions.access" not in message:
            raise RuntimeError
    else:
        response.close()
        raise RuntimeError
except Exception:
    raise SystemExit("administrator secret was not the exact expected metadata-identity IAM denial") from None

print(json.dumps({
    "administrator_secret_access": "iam_permission_denied",
    "credential_context": "systemd_main_pid_environment",
    "identity_source": "gce_metadata",
    "stdout_discarded": True,
}, sort_keys=True))
PY
}

run_scoped_status() {
  local phase="$1" tool_path="$2"
  assert_attached_service_account
  echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical"
  sudo -n -u teleo env -i \
    HOME=/home/teleo \
    PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
    TELEO_GCP_PROJECT=teleo-501523 \
    TELEO_GCP_METADATA_ONLY=1 \
    TELEO_CLOUDSQL_HOST=10.61.0.3 \
    TELEO_CLOUDSQL_PORT=5432 \
    TELEO_CLOUDSQL_DB=teleo_canonical \
    TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical \
    TELEO_CLOUDSQL_USER=leoclean_kb_runtime \
    TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \
    TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \
    /usr/bin/python3 "$tool_path" status --format json --redacted
}

run_service_wrapper_status() {
  local phase="$1" main_pid
  assert_attached_service_account
  main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
  echo "SCOPED_WRAPPER_STATUS phase=$phase user=teleo database=teleo_canonical"
  # --env imports the target MainPID's effective /proc environment. setpriv
  # drops back to the service's Unix identity without sudo resetting that env.
  sudo nsenter -t "$main_pid" -m --env -- \
    /usr/bin/setpriv \
      --reuid=teleo \
      --regid=teleo \
      --init-groups \
      --no-new-privs \
      "$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted
}

service_fingerprint() {
  printf '%s|%s|%s|%s' \
    "$(systemctl show "$SERVICE" --property=ActiveState --value)" \
    "$(systemctl show "$SERVICE" --property=SubState --value)" \
    "$(systemctl show "$SERVICE" --property=MainPID --value)" \
    "$(systemctl show "$SERVICE" --property=NRestarts --value)"
}

assert_service_fingerprint() {
  local expected="$1" actual
  actual="$(service_fingerprint)"
  if [ "$actual" != "$expected" ]; then
    echo "Service state changed during the read-only preflight." >&2
    return 1
  fi
  echo "PREFLIGHT_SERVICE_UNCHANGED fingerprint=$actual"
}

run_permission_verifier() {
  local phase="$1" verifier_path="$2" receipt_path="$3" receipt_dir run_id receipt_state
  run_id="$(date -u +%Y%m%d%H%M%S)-$phase"
  receipt_dir="$(dirname "$receipt_path")"
  sudo -n -u teleo mkdir -p -- "$receipt_dir"
  sudo -n -u teleo chmod 0700 "$receipt_dir"
  echo "NEGATIVE_PERMISSION_RECEIPT phase=$phase run_id=$run_id"
  sudo -n -u teleo env -i \
    HOME=/home/teleo \
    PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
    /usr/bin/python3 "$verifier_path" --run-id "$run_id" --output "$receipt_path"
  receipt_state="$(sudo stat -c '%U:%G:%a' "$receipt_path")"
  if [ "$receipt_state" != "teleo:teleo:600" ]; then
    echo "Permission receipt has unexpected ownership or mode: $receipt_state" >&2
    return 1
  fi
  sudo /usr/bin/python3 - "$receipt_path" <<'PY'
import json
import sys
from pathlib import Path

receipt = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
expected = {
    "artifact": "gcp_leoclean_runtime_permissions",
    "current_tier": "T3_live_readonly",
    "mode": "live_private_gcp_staging",
    "required_tier": "T3_live_readonly",
    "status": "pass",
}
mismatched = sorted(key for key, value in expected.items() if receipt.get(key) != value)
if mismatched:
    raise SystemExit("permission receipt contract mismatch: " + ",".join(mismatched))
print(json.dumps({"permission_receipt": "validated", "validated_fields": sorted(expected)}, sort_keys=True))
PY
}

assert_installed_files() {
  local wrapper tool permission_verifier service_env_verifier dropin revision executable_dir protected_dir protected_file
  wrapper="$REMOTE_RUNTIME_BIN/teleo-kb"
  tool="$REMOTE_RUNTIME_BIN/cloudsql_memory_tool.py"
  permission_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py"
  service_env_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"
  dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME"
  revision="$REMOTE_RUNTIME_BIN/.livingip-runtime-revision"
  test "$(sudo sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA"
  test "$(sudo sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA"
  test "$(sudo sha256sum "$permission_verifier" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
  test "$(sudo sha256sum "$service_env_verifier" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
  test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA"
  test "$(sudo stat -c '%U:%G:%a' "$REMOTE_RUNTIME_BIN")" = "root:root:755"
  test "$(sudo stat -c '%U:%G:%a' "$wrapper")" = "root:root:755"
  test "$(sudo stat -c '%U:%G:%a' "$tool")" = "root:root:755"
  test "$(sudo stat -c '%U:%G:%a' "$permission_verifier")" = "root:root:755"
  test "$(sudo stat -c '%U:%G:%a' "$service_env_verifier")" = "root:root:755"
  test "$(sudo stat -c '%U:%G:%a' "$revision")" = "root:root:644"
  test "$(sudo stat -c '%U:%G:%a' "$dropin")" = "root:root:644"
  test "$(sudo cat "$revision")" = "$SOURCE_REVISION"
  for protected_dir in \
    /usr \
    /usr/local \
    /usr/local/libexec \
    /usr/local/libexec/livingip \
    "$REMOTE_RUNTIME_BIN" \
    "$REMOTE_DROPIN_DIR"; do
    sudo test -d "$protected_dir"
    sudo test ! -L "$protected_dir"
    test "$(sudo stat -c '%U:%G' "$protected_dir")" = "root:root"
    sudo -n -u teleo test ! -w "$protected_dir"
  done
  for protected_file in \
    "$wrapper" \
    "$tool" \
    "$permission_verifier" \
    "$service_env_verifier" \
    "$revision" \
    "$dropin"; do
    sudo test -f "$protected_file"
    sudo test ! -L "$protected_file"
    sudo -n -u teleo test ! -w "$protected_file"
  done
  for executable_dir in \
    /usr/local/sbin \
    /usr/local/bin \
    /usr/sbin \
    /usr/bin \
    /sbin \
    /bin; do
    sudo test -d "$executable_dir"
    test "$(sudo stat -Lc '%U:%G' "$executable_dir")" = "root:root"
    sudo -n -u teleo test ! -w "$executable_dir"
  done
  for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3; do
    sudo test -x "$protected_file"
    test "$(sudo stat -Lc '%U:%G' "$protected_file")" = "root:root"
    sudo -n -u teleo test ! -w "$protected_file"
  done
  sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin"
  sudo grep -E '^Environment=TELEO_(KB_MODE|GCP_PROJECT|GCP_METADATA_ONLY|CLOUDSQL_|CANONICAL_CLOUDSQL_DB)' "$dropin"
  sudo grep -Fx "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$dropin"
  sudo grep -Fx "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$dropin"
  sudo grep -Fx "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$dropin"
  sudo grep -Fx 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$dropin"
}

assert_deployed_files() {
  local main_pid namespace_state source_state mount_state
  local hermes_mount_state state_mount_state workspace_mount_state capability_set
  assert_installed_files

  main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
  source_state="$(stat -Lc '%d:%i' "$REMOTE_RUNTIME_BIN")"
  namespace_state="$(sudo nsenter -t "$main_pid" -m -- stat -Lc '%d:%i' "$SERVICE_PROFILE_BIN")"
  if [ "$namespace_state" != "$source_state" ]; then
    echo "Service namespace does not bind the reviewed runtime directory." >&2
    return 1
  fi
  mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -T "$SERVICE_PROFILE_BIN" -n -o OPTIONS)"
  case ",$mount_state," in
    *,ro,*) ;;
    *)
      echo "Service runtime bind is not read-only." >&2
      return 1
      ;;
  esac
  hermes_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_HERMES_ROOT" -n -o OPTIONS)"
  state_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/state" -n -o OPTIONS)"
  workspace_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/workspace" -n -o OPTIONS)"
  case ",$hermes_mount_state," in
    *,ro,*) ;;
    *)
      echo "Service .hermes ancestor mount is not read-only." >&2
      return 1
      ;;
  esac
  case ",$state_mount_state," in
    *,rw,*) ;;
    *)
      echo "Service state mount is not writable." >&2
      return 1
      ;;
  esac
  case ",$workspace_mount_state," in
    *,rw,*) ;;
    *)
      echo "Service workspace mount is not writable." >&2
      return 1
      ;;
  esac
  sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w /home
  sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT"
  sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT/profiles"
  sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_ROOT"
  sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_BIN"
  sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/state"
  sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/workspace"
  capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)"
  if printf '%s\n' "$capability_set" | grep -Eqi '(^|[[:space:]])cap_sys_admin([[:space:]]|$)'; then
    echo "Service capability bounding set still contains CAP_SYS_ADMIN." >&2
    return 1
  fi
  test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb" | awk '{print $1}')" = "$WRAPPER_SHA"
  test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql_memory_tool.py" | awk '{print $1}')" = "$TOOL_SHA"
  test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_runtime_permissions.py" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
  test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_service_environment.py" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
}
REMOTE
)

printf -v verify_context \
  'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \
  "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \
  "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \
  "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" "$SOURCE_REVISION"

verify_body=$(cat <<'REMOTE'
set -euo pipefail
verification_tmp="$(mktemp -d /tmp/teleo-gcp-runtime-verify.XXXXXX)"
sudo chown teleo:teleo "$verification_tmp"
sudo chmod 0700 "$verification_tmp"
cleanup_verification() { sudo rm -rf -- "$verification_tmp"; }
trap cleanup_verification EXIT
assert_deployed_files
assert_service_running
assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"
run_service_wrapper_status verify
assert_administrator_secret_denied
run_permission_verifier verify "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" "$verification_tmp/permission-receipt.json"
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
REMOTE
)
verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}"

if $DRY_RUN; then
  echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE preflight_only=$PREFLIGHT_ONLY restart=$RESTART"
  echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA dropin_sha256=$DROPIN_SHA"
  exit 0
fi

if $VERIFY_ONLY; then
  "${GCP_SSH[@]}" --command="$verify_command"
  exit 0
fi

if ! $RESTART && ! $PREFLIGHT_ONLY; then
  echo "Refusing to install runtime files without --restart; use --dry-run, --preflight-only, or --verify-only for read-only work." >&2
  exit 64
fi

printf -v sync_context \
  'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \
  "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \
  "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \
  "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" \
  "$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY"

sync_body=$(cat <<'REMOTE'
set -euo pipefail
remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)"
runtime_bin="$REMOTE_RUNTIME_BIN"
runtime_parent="$(dirname "$runtime_bin")"
runtime_grandparent="$(dirname "$runtime_parent")"
runtime_anchor="$(dirname "$runtime_grandparent")"
dropin_dir="$REMOTE_DROPIN_DIR"
dropin_parent="$(dirname "$dropin_dir")"
payload_dir="$remote_tmp/payload"
backup_dir="$remote_tmp/backup"
work_dir="$remote_tmp/work"
wrapper_stage="$runtime_bin/.teleo-kb.livingip-new.$$"
tool_stage="$runtime_bin/.cloudsql-memory-tool.livingip-new.$$"
permission_verifier_stage="$runtime_bin/.runtime-permission-verifier.livingip-new.$$"
service_env_verifier_stage="$runtime_bin/.service-environment-verifier.livingip-new.$$"
revision_stage="$runtime_bin/.livingip-runtime-revision.livingip-new.$$"
dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$"
mutation_started=false

assert_existing_root_directory() {
  local target="$1"
  if sudo test -e "$target" || sudo test -L "$target"; then
    sudo test -d "$target"
    sudo test ! -L "$target"
    test "$(sudo stat -c '%U:%G' "$target")" = "root:root"
    sudo -n -u teleo test ! -w "$target"
  fi
}

assert_existing_root_file() {
  local target="$1"
  if sudo test -e "$target" || sudo test -L "$target"; then
    sudo test -f "$target"
    sudo test ! -L "$target"
    test "$(sudo stat -c '%U:%G' "$target")" = "root:root"
    sudo -n -u teleo test ! -w "$target"
  fi
}

backup_path() {
  local target="$1" name="$2"
  if sudo test -e "$target"; then
    sudo cp -a -- "$target" "$backup_dir/$name"
    sudo touch "$backup_dir/$name.present"
  fi
}

backup_directory_state() {
  local target="$1" name="$2"
  if sudo test -d "$target"; then
    sudo stat -c '%u %g %a' "$target" | sudo tee "$backup_dir/$name.metadata" >/dev/null
    sudo touch "$backup_dir/$name.present"
  fi
}

restore_directory_state() {
  local target="$1" name="$2" uid gid mode
  if sudo test -f "$backup_dir/$name.present"; then
    read -r uid gid mode <<<"$(sudo cat "$backup_dir/$name.metadata")"
    sudo chown "$uid:$gid" "$target"
    sudo chmod "$mode" "$target"
  elif sudo test -d "$target"; then
    sudo rmdir -- "$target"
  fi
}

restore_path() {
  local target="$1" name="$2" restore_rc=0 restore_tmp
  if sudo test -f "$backup_dir/$name.present"; then
    restore_tmp="${target}.livingip-rollback.$$"
    sudo rm -f -- "$restore_tmp" || restore_rc=1
    if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then
      sudo mv -f -- "$restore_tmp" "$target" || restore_rc=1
    else
      restore_rc=1
    fi
    sudo rm -f -- "$restore_tmp" || restore_rc=1
  else
    sudo rm -f -- "$target" || restore_rc=1
  fi
  return "$restore_rc"
}

validate_restored_path() {
  local target="$1" name="$2"
  if sudo test -f "$backup_dir/$name.present"; then
    sudo test -f "$target"
    sudo test ! -L "$target"
    sudo cmp -s -- "$backup_dir/$name" "$target"
    test "$(sudo stat -c '%u:%g:%a' "$target")" = \
      "$(sudo stat -c '%u:%g:%a' "$backup_dir/$name")"
  else
    sudo test ! -e "$target"
    sudo test ! -L "$target"
  fi
}

validate_restored_directory() {
  local target="$1" name="$2" expected_state
  if sudo test -f "$backup_dir/$name.present"; then
    expected_state="$(sudo cat "$backup_dir/$name.metadata" | tr ' ' ':')"
    sudo test -d "$target"
    sudo test ! -L "$target"
    test "$(sudo stat -c '%u:%g:%a' "$target")" = "$expected_state"
  else
    sudo test ! -e "$target"
    sudo test ! -L "$target"
  fi
}

rollback_runtime() {
  local rollback_rc=0
  echo "Deployment failed; restoring the complete previous runtime set." >&2
  sudo systemctl stop "$SERVICE" || rollback_rc=1
  restore_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1
  restore_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
  restore_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1
  restore_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1
  restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1
  restore_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1
  restore_directory_state "$runtime_bin" runtime_bin || rollback_rc=1
  restore_directory_state "$runtime_parent" runtime_parent || rollback_rc=1
  restore_directory_state "$runtime_grandparent" runtime_grandparent || rollback_rc=1
  restore_directory_state "$dropin_dir" dropin_dir || rollback_rc=1
  if [ "$rollback_rc" -eq 0 ]; then
    validate_restored_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1
    validate_restored_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
    validate_restored_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1
    validate_restored_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1
    validate_restored_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1
    validate_restored_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1
    validate_restored_directory "$runtime_grandparent" runtime_grandparent || rollback_rc=1
    validate_restored_directory "$runtime_parent" runtime_parent || rollback_rc=1
    validate_restored_directory "$runtime_bin" runtime_bin || rollback_rc=1
    validate_restored_directory "$dropin_dir" dropin_dir || rollback_rc=1
  fi
  if [ "$rollback_rc" -eq 0 ]; then
    sudo systemctl daemon-reload || rollback_rc=1
  fi
  if [ "$rollback_rc" -eq 0 ]; then
    assert_unit_configuration false || rollback_rc=1
  fi
  if [ "$rollback_rc" -eq 0 ]; then
    sudo systemctl start "$SERVICE" || rollback_rc=1
    assert_service_running || rollback_rc=1
  fi
  return "$rollback_rc"
}

on_exit() {
  local rc="$1" rollback_rc=0
  trap - EXIT INT TERM
  if [ "$rc" -ne 0 ] && [ "$mutation_started" = true ]; then
    set +e
    rollback_runtime
    rollback_rc=$?
    set -e
    if [ "$rollback_rc" -ne 0 ]; then
      echo "CRITICAL: previous runtime restoration did not return the service to active/running." >&2
      rc=1
    fi
  fi
  sudo rm -f -- \
    "$wrapper_stage" \
    "$tool_stage" \
    "$permission_verifier_stage" \
    "$service_env_verifier_stage" \
    "$revision_stage" \
    "$dropin_stage" || true
  sudo rm -rf -- "$remote_tmp" || true
  exit "$rc"
}

trap 'on_exit $?' EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

sudo chmod 0711 "$remote_tmp"
sudo install -d -o root -g root -m 0755 "$payload_dir"
sudo install -d -o root -g root -m 0700 "$backup_dir"
sudo install -d -o teleo -g teleo -m 0700 "$work_dir"
sudo tar --no-same-owner --no-same-permissions --no-xattrs --no-acls --no-selinux -C "$payload_dir" -xf -
if sudo find "$payload_dir" -type l -print -quit | grep -q .; then
  echo "Staged runtime payload contains a symlink." >&2
  exit 65
fi
if sudo find "$payload_dir" ! -type d ! -type f -print -quit | grep -q .; then
  echo "Staged runtime payload contains a non-regular entry." >&2
  exit 65
fi
sudo chown -R root:root "$payload_dir"
sudo find "$payload_dir" -type d -exec chmod 0555 {} +
sudo find "$payload_dir" -type f -exec chmod 0444 {} +
sudo chmod 0555 \
  "$payload_dir/$WRAPPER_REL" \
  "$payload_dir/$TOOL_REL" \
  "$payload_dir/$PERMISSION_VERIFIER_REL" \
  "$payload_dir/$SERVICE_ENV_VERIFIER_REL"
test "$(sudo sha256sum "$payload_dir/$WRAPPER_REL" | awk '{print $1}')" = "$WRAPPER_SHA"
test "$(sudo sha256sum "$payload_dir/$TOOL_REL" | awk '{print $1}')" = "$TOOL_SHA"
test "$(sudo sha256sum "$payload_dir/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
test "$(sudo sha256sum "$payload_dir/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
test "$(sudo sha256sum "$payload_dir/$DROPIN_REL" | awk '{print $1}')" = "$DROPIN_SHA"
test "$(sudo stat -c '%U:%G:%a' "$remote_tmp")" = "root:root:711"
test "$(sudo stat -c '%U:%G:%a' "$payload_dir")" = "root:root:555"
test "$(sudo stat -c '%U:%G:%a' "$backup_dir")" = "root:root:700"
test "$(sudo stat -c '%U:%G:%a' "$work_dir")" = "teleo:teleo:700"
sudo -n -u teleo test ! -w "$remote_tmp"
sudo -n -u teleo test ! -w "$payload_dir"
sudo -n -u teleo test ! -w "$backup_dir"
sudo test -d "$runtime_anchor"
sudo test -d "$dropin_parent"
sudo test ! -L "$runtime_anchor"
sudo test ! -L "$dropin_parent"
test "$(sudo stat -c '%U:%G' "$runtime_anchor")" = "root:root"
test "$(sudo stat -c '%U:%G' "$dropin_parent")" = "root:root"
sudo -n -u teleo test ! -w "$runtime_anchor"
sudo -n -u teleo test ! -w "$dropin_parent"
for existing_directory in \
  "$runtime_grandparent" \
  "$runtime_parent" \
  "$runtime_bin" \
  "$dropin_dir"; do
  assert_existing_root_directory "$existing_directory"
done
for existing_file in \
  "$runtime_bin/teleo-kb" \
  "$runtime_bin/cloudsql_memory_tool.py" \
  "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" \
  "$runtime_bin/verify_gcp_leoclean_service_environment.py" \
  "$runtime_bin/.livingip-runtime-revision" \
  "$dropin_dir/$DROPIN_NAME"; do
  assert_existing_root_file "$existing_file"
done

assert_service_running
preflight_service_before="$(service_fingerprint)"
assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false
run_scoped_status pre "$payload_dir/$TOOL_REL"
assert_administrator_secret_denied
run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json"
assert_service_fingerprint "$preflight_service_before"
if [ "$PREFLIGHT_ONLY" = true ]; then
  systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
  exit 0
fi

backup_path "$runtime_bin/teleo-kb" wrapper
backup_path "$runtime_bin/cloudsql_memory_tool.py" tool
backup_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier
backup_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier
backup_path "$dropin_dir/$DROPIN_NAME" dropin
backup_path "$runtime_bin/.livingip-runtime-revision" revision
backup_directory_state "$runtime_parent" runtime_parent
backup_directory_state "$runtime_grandparent" runtime_grandparent
backup_directory_state "$runtime_bin" runtime_bin
backup_directory_state "$dropin_dir" dropin_dir

mutation_started=true
sudo systemctl stop "$SERVICE"
sudo install -d -o root -g root -m 0755 "$runtime_parent"
sudo install -d -o root -g root -m 0755 "$runtime_bin"
sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage"
sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage"
sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" "$permission_verifier_stage"
sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" "$service_env_verifier_stage"
sudo mv -f -- "$wrapper_stage" "$runtime_bin/teleo-kb"
sudo mv -f -- "$tool_stage" "$runtime_bin/cloudsql_memory_tool.py"
sudo mv -f -- "$permission_verifier_stage" "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py"
sudo mv -f -- "$service_env_verifier_stage" "$runtime_bin/verify_gcp_leoclean_service_environment.py"
sudo install -d -o root -g root -m 0755 "$dropin_dir"
sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL" "$dropin_stage"
sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME"
printf '%s\n' "$SOURCE_REVISION" | sudo tee "$revision_stage" >/dev/null
sudo chown root:root "$revision_stage"
sudo chmod 0644 "$revision_stage"
sudo mv -f -- "$revision_stage" "$runtime_bin/.livingip-runtime-revision"
assert_installed_files
sudo systemctl daemon-reload
assert_unit_configuration
sudo systemctl start "$SERVICE"

assert_deployed_files
assert_service_running
assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py"
run_service_wrapper_status post
assert_administrator_secret_denied
run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" "$work_dir/permission-post/receipt.json"
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
mutation_started=false
REMOTE
)
sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}"

COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \
  "$WRAPPER_REL" \
  "$TOOL_REL" \
  "$PERMISSION_VERIFIER_REL" \
  "$SERVICE_ENV_VERIFIER_REL" \
  "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command"