From b7bdcea46acdf9ddff85c56ce6bda3837a576752 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 04:40:26 +0200 Subject: [PATCH] fix: enforce scoped GCP Leo runtime credentials --- .github/workflows/ci.yml | 7 + deploy/sync-gcp-leoclean-runtime.sh | 696 ++++++- docs/gcp-leoclean-runtime-reconciliation.md | 144 +- .../leoclean-bin/cloudsql_memory_tool.py | 291 ++- hermes-agent/leoclean-bin/teleo-kb | 37 +- ops/gcp_leoclean_runtime_role.sql | 53 +- ...verify_gcp_leoclean_runtime_permissions.py | 1838 +++++++++++++++++ ...verify_gcp_leoclean_service_environment.py | 219 ++ ...run_gcp_generated_db_direct_claim_suite.py | 2 +- .../leoclean-gcp-prod-parallel-cloudsql.conf | 16 +- .../test_gcp_leoclean_runtime_permissions.py | 797 +++++++ .../test_gcp_leoclean_service_environment.py | 281 +++ .../test_hermes_leoclean_kb_bridge_source.py | 523 ++++- tests/test_teleo_agent_systemd.py | 351 +++- 14 files changed, 5031 insertions(+), 224 deletions(-) create mode 100644 ops/verify_gcp_leoclean_runtime_permissions.py create mode 100755 ops/verify_gcp_leoclean_service_environment.py create mode 100644 tests/test_gcp_leoclean_runtime_permissions.py create mode 100644 tests/test_gcp_leoclean_service_environment.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91dada8..3367127 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,8 @@ jobs: ops/restore_gcp_generated_postgres_snapshot.py \ ops/sqlite_to_postgres_dump.py \ ops/verify_gcp_cloudsql_restore_readback.py \ + ops/verify_gcp_leoclean_runtime_permissions.py \ + ops/verify_gcp_leoclean_service_environment.py \ ops/verify_postgres_parity_manifest.py \ telegram/approvals.py \ hermes-agent/leoclean-bin/kb_tool.py \ @@ -80,6 +82,8 @@ jobs: tests/test_capture_vps_canonical_postgres_snapshot.py \ tests/test_gcp_infra_execute_canary.py \ tests/test_gcp_infra_readiness_checker.py \ + tests/test_gcp_leoclean_runtime_permissions.py \ + tests/test_gcp_leoclean_service_environment.py \ tests/test_gcp_runtime_baseline_apply.py \ tests/test_gcp_service_communications.py \ tests/test_gcp_cloudsql_restore_drill.py \ @@ -102,12 +106,15 @@ jobs: tests/test_restore_gcp_generated_postgres_snapshot.py \ tests/test_sqlite_to_postgres_dump.py \ tests/test_sqlite_postgres_restore_canary_capsule.py \ + tests/test_teleo_agent_systemd.py \ tests/test_eval_parse.py \ tests/test_contributor.py \ tests/test_search.py - name: Shell syntax run: | bash -n \ + deploy/sync-gcp-leoclean-runtime.sh \ + hermes-agent/leoclean-bin/teleo-kb \ ops/backup_vps_sqlite_kb.sh \ ops/run_gcp_cloudsql_restore_drill.sh \ ops/run_sqlite_postgres_restore_canary.sh diff --git a/deploy/sync-gcp-leoclean-runtime.sh b/deploy/sync-gcp-leoclean-runtime.sh index 3661155..25a7976 100755 --- a/deploy/sync-gcp-leoclean-runtime.sh +++ b/deploy/sync-gcp-leoclean-runtime.sh @@ -10,23 +10,28 @@ ZONE="${ZONE:-europe-west6-a}" INSTANCE="${INSTANCE:-teleo-prod-1}" SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}" MERGED_REF="${MERGED_REF:-origin/main}" -REMOTE_PROFILE_BIN="${REMOTE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}" +REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}" +SERVICE_PROFILE_BIN="${SERVICE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}" +SERVICE_PROFILE_ROOT="${SERVICE_PROFILE_ROOT:-/home/teleo/.hermes/profiles/leoclean}" +SERVICE_HERMES_ROOT="${SERVICE_HERMES_ROOT:-/home/teleo/.hermes}" REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}" -REMOTE_CLOUDSDK_CONFIG="${REMOTE_CLOUDSDK_CONFIG:-/run/leoclean-gcp-prod-parallel}" EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}" WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb" TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py" +PERMISSION_VERIFIER_REL="ops/verify_gcp_leoclean_runtime_permissions.py" +SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py" DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf" DROPIN_NAME="10-cloudsql-memory.conf" DRY_RUN=false RESTART=false VERIFY_ONLY=false +PREFLIGHT_ONLY=false usage() { cat <<'USAGE' -Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--restart] [--verify-only] +Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--preflight-only] [--restart] [--verify-only] The database role and scoped Secret Manager secret must exist before --restart. This script deploys only the bridge files and non-secret systemd drop-in. @@ -36,6 +41,7 @@ USAGE for arg in "$@"; do case "$arg" in --dry-run) DRY_RUN=true ;; + --preflight-only) PREFLIGHT_ONLY=true ;; --restart) RESTART=true ;; --verify-only) VERIFY_ONLY=true ;; --help|-h) @@ -50,28 +56,72 @@ for arg in "$@"; do esac done -for path in "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; do +if $PREFLIGHT_ONLY && $RESTART; then + echo "--preflight-only and --restart are mutually exclusive." >&2 + exit 64 +fi +if $VERIFY_ONLY && { $PREFLIGHT_ONLY || $RESTART; }; then + echo "--verify-only cannot be combined with --preflight-only or --restart." >&2 + exit 64 +fi + +for path in \ + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$DROPIN_REL"; do if [ ! -f "$REPO_ROOT/$path" ]; then echo "Required source file is missing: $path" >&2 exit 66 fi + if [ -L "$REPO_ROOT/$path" ]; then + echo "Refusing symlinked runtime source file: $path" >&2 + exit 65 + fi done if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2 exit 65 fi -if ! grep -Fxq "Environment=CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG" "$REPO_ROOT/$DROPIN_REL"; then - echo "Runtime drop-in CLOUDSDK_CONFIG does not match $REMOTE_CLOUDSDK_CONFIG." >&2 +if ! grep -Fxq 'Environment=TELEO_GCP_METADATA_ONLY=1' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must enforce metadata-only GCP credentials." >&2 + exit 65 +fi +if ! grep -Fxq "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not bind the root-controlled runtime into the service profile." >&2 + exit 65 +fi +if ! grep -Fxq "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not protect the service-profile ancestor from replacement." >&2 + exit 65 +fi +if ! grep -Fxq "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not limit service-profile writes to state and workspace." >&2 + exit 65 +fi +if ! grep -Fxq 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must remove CAP_SYS_ADMIN from the service." >&2 exit 65 fi SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)" -if ! git -C "$REPO_ROOT" diff --quiet HEAD -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; then +if ! git -C "$REPO_ROOT" diff --quiet HEAD -- \ + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$DROPIN_REL"; then echo "Refusing to deploy runtime files that do not match HEAD." >&2 exit 65 fi -if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL")" ]; then +if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- \ + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$DROPIN_REL")" ]; then echo "Refusing to deploy untracked runtime files." >&2 exit 65 fi @@ -85,6 +135,8 @@ if ! git -C "$REPO_ROOT" merge-base --is-ancestor "$SOURCE_REVISION" "$MERGED_RE fi WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')" TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')" +PERMISSION_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" +SERVICE_ENV_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')" GCP_SSH=( @@ -123,64 +175,132 @@ assert_service_running() { fi } -assert_runtime_environment() { - local environment runtime_dir_state - environment="$(systemctl show "$SERVICE" --property=Environment --value)" - case " $environment " in - *" PGPASSWORD="*) - echo "Service environment still contains PGPASSWORD." >&2 +assert_unit_configuration() { + local require_reviewed="${1:-true}" + local dropin_path dropin_name dropin_paths environment_files reviewed_dropin_seen=false + dropin_paths="$(systemctl show "$SERVICE" --property=DropInPaths --value)" + environment_files="$(systemctl show "$SERVICE" --property=EnvironmentFiles --value)" + if [ -n "$environment_files" ]; then + echo "Service has an effective EnvironmentFile, which is not allowed for the scoped runtime." >&2 + return 1 + fi + for dropin_path in $dropin_paths; do + if [ "$dropin_path" = "$REMOTE_DROPIN_DIR/$DROPIN_NAME" ]; then + reviewed_dropin_seen=true + continue + fi + dropin_name="${dropin_path##*/}" + if [[ "$dropin_name" > "$DROPIN_NAME" ]]; then + echo "Service has a later drop-in than the reviewed Cloud SQL configuration: $dropin_name" >&2 return 1 - ;; - esac - case " $environment " in - *" CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG "*) ;; - *) - echo "Service does not use the isolated CLOUDSDK_CONFIG." >&2 - return 1 - ;; - esac - runtime_dir_state="$(stat -c '%U:%G:%a' "$REMOTE_CLOUDSDK_CONFIG")" - if [ "$runtime_dir_state" != "teleo:teleo:700" ]; then - echo "Isolated CLOUDSDK_CONFIG has unexpected ownership or mode: $runtime_dir_state" >&2 + fi + done + if [ "$require_reviewed" = true ] && [ "$reviewed_dropin_seen" != true ]; then + echo "Service does not load the reviewed Cloud SQL drop-in." >&2 return 1 fi } +assert_runtime_environment() { + local verifier_path="$1" require_reviewed="${2:-true}" main_pid + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + sudo /usr/bin/python3 "$verifier_path" --pid "$main_pid" + assert_unit_configuration "$require_reviewed" +} + assert_attached_service_account() { - local sdk_config="$1" metadata_email - sudo install -d -o teleo -g teleo -m 0700 "$sdk_config" + local metadata_email metadata_email="$( - sudo -n -u teleo env -u PGPASSWORD \ + sudo -n -u teleo env -i \ HOME=/home/teleo \ - CLOUDSDK_CONFIG="$sdk_config" \ - curl --fail --silent --show-error \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + /usr/bin/curl -q --noproxy '*' --proto '=http' --max-time 10 --fail --silent --show-error \ --header 'Metadata-Flavor: Google' \ - 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email' + 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email' )" if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then echo "Attached VM service account mismatch: $metadata_email" >&2 return 1 fi - - # An empty, phase-local Cloud SDK config forces this token request to use - # the VM's attached identity instead of an operator's cached credentials. - sudo -n -u teleo env -u PGPASSWORD \ - HOME=/home/teleo \ - CLOUDSDK_CONFIG="$sdk_config" \ - CLOUDSDK_CORE_PROJECT="$PROJECT" \ - gcloud --quiet auth print-access-token >/dev/null echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email" } +assert_administrator_secret_denied() { + local main_pid + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + sudo nsenter -t "$main_pid" -m --env -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --init-groups \ + --no-new-privs \ + /usr/bin/python3 - "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" <<'PY' +import json +import sys +import urllib.error +import urllib.parse +import urllib.request + +project, expected_email = sys.argv[1:] +metadata_headers = {"Metadata-Flavor": "Google"} +opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def metadata(path: str) -> bytes: + request = urllib.request.Request( + "http://169.254.169.254/computeMetadata/v1/" + path, + headers=metadata_headers, + ) + with opener.open(request, timeout=10) as response: + return response.read(65536) + + +try: + email = metadata("instance/service-accounts/default/email").decode("utf-8").strip() + if email != expected_email: + raise RuntimeError + token_payload = json.loads(metadata("instance/service-accounts/default/token")) + token = token_payload.get("access_token") + if token_payload.get("token_type") != "Bearer" or not isinstance(token, str) or not token: + raise RuntimeError + secret = urllib.parse.quote("gcp-teleo-pgvector-standby-postgres-password", safe="") + url = f"https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{secret}/versions/latest:access" + request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + try: + response = opener.open(request, timeout=10) + except urllib.error.HTTPError as error: + body = error.read(65536) + parsed = json.loads(body) + detail = parsed.get("error", {}) + message = str(detail.get("message", "")).lower() + if error.code != 403 or detail.get("status") != "PERMISSION_DENIED": + raise RuntimeError from None + if "secretmanager.versions.access" not in message: + raise RuntimeError + else: + response.close() + raise RuntimeError +except Exception: + raise SystemExit("administrator secret was not the exact expected metadata-identity IAM denial") from None + +print(json.dumps({ + "administrator_secret_access": "iam_permission_denied", + "credential_context": "systemd_main_pid_environment", + "identity_source": "gce_metadata", + "stdout_discarded": True, +}, sort_keys=True)) +PY +} + run_scoped_status() { - local phase="$1" tool_path="$2" sdk_config="$3" - assert_attached_service_account "$sdk_config" + local phase="$1" tool_path="$2" + assert_attached_service_account echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical" - sudo -n -u teleo env -u PGPASSWORD \ + sudo -n -u teleo env -i \ HOME=/home/teleo \ - CLOUDSDK_CONFIG="$sdk_config" \ - CLOUDSDK_CORE_PROJECT="$PROJECT" \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ TELEO_GCP_PROJECT=teleo-501523 \ + TELEO_GCP_METADATA_ONLY=1 \ TELEO_CLOUDSQL_HOST=10.61.0.3 \ TELEO_CLOUDSQL_PORT=5432 \ TELEO_CLOUDSQL_DB=teleo_canonical \ @@ -188,46 +308,238 @@ run_scoped_status() { TELEO_CLOUDSQL_USER=leoclean_kb_runtime \ TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \ TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \ - python3 "$tool_path" status --format json --redacted + /usr/bin/python3 "$tool_path" status --format json --redacted +} + +run_service_wrapper_status() { + local phase="$1" main_pid + assert_attached_service_account + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + echo "SCOPED_WRAPPER_STATUS phase=$phase user=teleo database=teleo_canonical" + # --env imports the target MainPID's effective /proc environment. setpriv + # drops back to the service's Unix identity without sudo resetting that env. + sudo nsenter -t "$main_pid" -m --env -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --init-groups \ + --no-new-privs \ + "$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted +} + +service_fingerprint() { + printf '%s|%s|%s|%s' \ + "$(systemctl show "$SERVICE" --property=ActiveState --value)" \ + "$(systemctl show "$SERVICE" --property=SubState --value)" \ + "$(systemctl show "$SERVICE" --property=MainPID --value)" \ + "$(systemctl show "$SERVICE" --property=NRestarts --value)" +} + +assert_service_fingerprint() { + local expected="$1" actual + actual="$(service_fingerprint)" + if [ "$actual" != "$expected" ]; then + echo "Service state changed during the read-only preflight." >&2 + return 1 + fi + echo "PREFLIGHT_SERVICE_UNCHANGED fingerprint=$actual" +} + +run_permission_verifier() { + local phase="$1" verifier_path="$2" receipt_path="$3" receipt_dir run_id receipt_state + run_id="$(date -u +%Y%m%d%H%M%S)-$phase" + receipt_dir="$(dirname "$receipt_path")" + sudo -n -u teleo mkdir -p -- "$receipt_dir" + sudo -n -u teleo chmod 0700 "$receipt_dir" + echo "NEGATIVE_PERMISSION_RECEIPT phase=$phase run_id=$run_id" + sudo -n -u teleo env -i \ + HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + /usr/bin/python3 "$verifier_path" --run-id "$run_id" --output "$receipt_path" + receipt_state="$(sudo stat -c '%U:%G:%a' "$receipt_path")" + if [ "$receipt_state" != "teleo:teleo:600" ]; then + echo "Permission receipt has unexpected ownership or mode: $receipt_state" >&2 + return 1 + fi + sudo /usr/bin/python3 - "$receipt_path" <<'PY' +import json +import sys +from pathlib import Path + +receipt = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +expected = { + "artifact": "gcp_leoclean_runtime_permissions", + "current_tier": "T3_live_readonly", + "mode": "live_private_gcp_staging", + "required_tier": "T3_live_readonly", + "status": "pass", +} +mismatched = sorted(key for key, value in expected.items() if receipt.get(key) != value) +if mismatched: + raise SystemExit("permission receipt contract mismatch: " + ",".join(mismatched)) +print(json.dumps({"permission_receipt": "validated", "validated_fields": sorted(expected)}, sort_keys=True)) +PY +} + +assert_installed_files() { + local wrapper tool permission_verifier service_env_verifier dropin revision executable_dir protected_dir protected_file + wrapper="$REMOTE_RUNTIME_BIN/teleo-kb" + tool="$REMOTE_RUNTIME_BIN/cloudsql_memory_tool.py" + permission_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" + service_env_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" + dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME" + revision="$REMOTE_RUNTIME_BIN/.livingip-runtime-revision" + test "$(sudo sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA" + test "$(sudo sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA" + test "$(sudo sha256sum "$permission_verifier" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" + test "$(sudo sha256sum "$service_env_verifier" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" + test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA" + test "$(sudo stat -c '%U:%G:%a' "$REMOTE_RUNTIME_BIN")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$wrapper")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$tool")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$permission_verifier")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$service_env_verifier")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$revision")" = "root:root:644" + test "$(sudo stat -c '%U:%G:%a' "$dropin")" = "root:root:644" + test "$(sudo cat "$revision")" = "$SOURCE_REVISION" + for protected_dir in \ + /usr \ + /usr/local \ + /usr/local/libexec \ + /usr/local/libexec/livingip \ + "$REMOTE_RUNTIME_BIN" \ + "$REMOTE_DROPIN_DIR"; do + sudo test -d "$protected_dir" + sudo test ! -L "$protected_dir" + test "$(sudo stat -c '%U:%G' "$protected_dir")" = "root:root" + sudo -n -u teleo test ! -w "$protected_dir" + done + for protected_file in \ + "$wrapper" \ + "$tool" \ + "$permission_verifier" \ + "$service_env_verifier" \ + "$revision" \ + "$dropin"; do + sudo test -f "$protected_file" + sudo test ! -L "$protected_file" + sudo -n -u teleo test ! -w "$protected_file" + done + for executable_dir in \ + /usr/local/sbin \ + /usr/local/bin \ + /usr/sbin \ + /usr/bin \ + /sbin \ + /bin; do + sudo test -d "$executable_dir" + test "$(sudo stat -Lc '%U:%G' "$executable_dir")" = "root:root" + sudo -n -u teleo test ! -w "$executable_dir" + done + for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3; do + sudo test -x "$protected_file" + test "$(sudo stat -Lc '%U:%G' "$protected_file")" = "root:root" + sudo -n -u teleo test ! -w "$protected_file" + done + sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin" + sudo grep -E '^Environment=TELEO_(KB_MODE|GCP_PROJECT|GCP_METADATA_ONLY|CLOUDSQL_|CANONICAL_CLOUDSQL_DB)' "$dropin" + sudo grep -Fx "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$dropin" + sudo grep -Fx "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$dropin" + sudo grep -Fx "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$dropin" + sudo grep -Fx 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$dropin" } assert_deployed_files() { - local wrapper tool dropin - wrapper="$REMOTE_PROFILE_BIN/teleo-kb" - tool="$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py" - dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME" - test "$(sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA" - test "$(sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA" - test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA" - sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin" - sudo grep -E '^Environment=(CLOUDSDK_CONFIG|TELEO_(KB_MODE|GCP_PROJECT|CLOUDSQL_|CANONICAL_CLOUDSQL_DB))' "$dropin" + local main_pid namespace_state source_state mount_state + local hermes_mount_state state_mount_state workspace_mount_state capability_set + assert_installed_files + + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + source_state="$(stat -Lc '%d:%i' "$REMOTE_RUNTIME_BIN")" + namespace_state="$(sudo nsenter -t "$main_pid" -m -- stat -Lc '%d:%i' "$SERVICE_PROFILE_BIN")" + if [ "$namespace_state" != "$source_state" ]; then + echo "Service namespace does not bind the reviewed runtime directory." >&2 + return 1 + fi + mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -T "$SERVICE_PROFILE_BIN" -n -o OPTIONS)" + case ",$mount_state," in + *,ro,*) ;; + *) + echo "Service runtime bind is not read-only." >&2 + return 1 + ;; + esac + hermes_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_HERMES_ROOT" -n -o OPTIONS)" + state_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/state" -n -o OPTIONS)" + workspace_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/workspace" -n -o OPTIONS)" + case ",$hermes_mount_state," in + *,ro,*) ;; + *) + echo "Service .hermes ancestor mount is not read-only." >&2 + return 1 + ;; + esac + case ",$state_mount_state," in + *,rw,*) ;; + *) + echo "Service state mount is not writable." >&2 + return 1 + ;; + esac + case ",$workspace_mount_state," in + *,rw,*) ;; + *) + echo "Service workspace mount is not writable." >&2 + return 1 + ;; + esac + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w /home + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT/profiles" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_ROOT" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_BIN" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/state" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/workspace" + capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)" + if printf '%s\n' "$capability_set" | grep -Eqi '(^|[[:space:]])cap_sys_admin([[:space:]]|$)'; then + echo "Service capability bounding set still contains CAP_SYS_ADMIN." >&2 + return 1 + fi + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb" | awk '{print $1}')" = "$WRAPPER_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql_memory_tool.py" | awk '{print $1}')" = "$TOOL_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_runtime_permissions.py" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_service_environment.py" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" } REMOTE ) printf -v verify_context \ - 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\n' \ - "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \ - "$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \ - "$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA" + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ + "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ + "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" "$SOURCE_REVISION" verify_body=$(cat <<'REMOTE' set -euo pipefail verification_tmp="$(mktemp -d /tmp/teleo-gcp-runtime-verify.XXXXXX)" +sudo chown teleo:teleo "$verification_tmp" +sudo chmod 0700 "$verification_tmp" cleanup_verification() { sudo rm -rf -- "$verification_tmp"; } trap cleanup_verification EXIT assert_deployed_files assert_service_running -assert_runtime_environment -run_scoped_status verify "$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py" "$verification_tmp/gcloud" +assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" +run_service_wrapper_status verify +assert_administrator_secret_denied +run_permission_verifier verify "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" "$verification_tmp/permission-receipt.json" systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager REMOTE ) verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}" if $DRY_RUN; then - echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE restart=$RESTART" - echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA dropin_sha256=$DROPIN_SHA" + echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE preflight_only=$PREFLIGHT_ONLY restart=$RESTART" + echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA dropin_sha256=$DROPIN_SHA" exit 0 fi @@ -236,39 +548,88 @@ if $VERIFY_ONLY; then exit 0 fi -if ! $RESTART; then - echo "Refusing to install runtime files without --restart; use --dry-run or --verify-only for read-only work." >&2 +if ! $RESTART && ! $PREFLIGHT_ONLY; then + echo "Refusing to install runtime files without --restart; use --dry-run, --preflight-only, or --verify-only for read-only work." >&2 exit 64 fi -REMOTE_TMP="/tmp/teleo-gcp-leoclean-runtime-$(date -u +%Y%m%dT%H%M%SZ)-$$" printf -v sync_context \ - 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\nREMOTE_TMP=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\n' \ - "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \ - "$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \ - "$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA" "$REMOTE_TMP" \ - "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" "$SOURCE_REVISION" + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ + "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ + "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" \ + "$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY" sync_body=$(cat <<'REMOTE' set -euo pipefail -remote_tmp="$REMOTE_TMP" -profile_bin="$REMOTE_PROFILE_BIN" +remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)" +runtime_bin="$REMOTE_RUNTIME_BIN" +runtime_parent="$(dirname "$runtime_bin")" +runtime_grandparent="$(dirname "$runtime_parent")" +runtime_anchor="$(dirname "$runtime_grandparent")" dropin_dir="$REMOTE_DROPIN_DIR" +dropin_parent="$(dirname "$dropin_dir")" +payload_dir="$remote_tmp/payload" backup_dir="$remote_tmp/backup" +work_dir="$remote_tmp/work" +wrapper_stage="$runtime_bin/.teleo-kb.livingip-new.$$" +tool_stage="$runtime_bin/.cloudsql-memory-tool.livingip-new.$$" +permission_verifier_stage="$runtime_bin/.runtime-permission-verifier.livingip-new.$$" +service_env_verifier_stage="$runtime_bin/.service-environment-verifier.livingip-new.$$" +revision_stage="$runtime_bin/.livingip-runtime-revision.livingip-new.$$" dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$" mutation_started=false +assert_existing_root_directory() { + local target="$1" + if sudo test -e "$target" || sudo test -L "$target"; then + sudo test -d "$target" + sudo test ! -L "$target" + test "$(sudo stat -c '%U:%G' "$target")" = "root:root" + sudo -n -u teleo test ! -w "$target" + fi +} + +assert_existing_root_file() { + local target="$1" + if sudo test -e "$target" || sudo test -L "$target"; then + sudo test -f "$target" + sudo test ! -L "$target" + test "$(sudo stat -c '%U:%G' "$target")" = "root:root" + sudo -n -u teleo test ! -w "$target" + fi +} + backup_path() { local target="$1" name="$2" if sudo test -e "$target"; then sudo cp -a -- "$target" "$backup_dir/$name" - touch "$backup_dir/$name.present" + sudo touch "$backup_dir/$name.present" + fi +} + +backup_directory_state() { + local target="$1" name="$2" + if sudo test -d "$target"; then + sudo stat -c '%u %g %a' "$target" | sudo tee "$backup_dir/$name.metadata" >/dev/null + sudo touch "$backup_dir/$name.present" + fi +} + +restore_directory_state() { + local target="$1" name="$2" uid gid mode + if sudo test -f "$backup_dir/$name.present"; then + read -r uid gid mode <<<"$(sudo cat "$backup_dir/$name.metadata")" + sudo chown "$uid:$gid" "$target" + sudo chmod "$mode" "$target" + elif sudo test -d "$target"; then + sudo rmdir -- "$target" fi } restore_path() { local target="$1" name="$2" restore_rc=0 restore_tmp - if [ -f "$backup_dir/$name.present" ]; then + if sudo test -f "$backup_dir/$name.present"; then restore_tmp="${target}.livingip-rollback.$$" sudo rm -f -- "$restore_tmp" || restore_rc=1 if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then @@ -283,15 +644,65 @@ restore_path() { return "$restore_rc" } +validate_restored_path() { + local target="$1" name="$2" + if sudo test -f "$backup_dir/$name.present"; then + sudo test -f "$target" + sudo test ! -L "$target" + sudo cmp -s -- "$backup_dir/$name" "$target" + test "$(sudo stat -c '%u:%g:%a' "$target")" = \ + "$(sudo stat -c '%u:%g:%a' "$backup_dir/$name")" + else + sudo test ! -e "$target" + sudo test ! -L "$target" + fi +} + +validate_restored_directory() { + local target="$1" name="$2" expected_state + if sudo test -f "$backup_dir/$name.present"; then + expected_state="$(sudo cat "$backup_dir/$name.metadata" | tr ' ' ':')" + sudo test -d "$target" + sudo test ! -L "$target" + test "$(sudo stat -c '%u:%g:%a' "$target")" = "$expected_state" + else + sudo test ! -e "$target" + sudo test ! -L "$target" + fi +} + rollback_runtime() { local rollback_rc=0 echo "Deployment failed; restoring the complete previous runtime set." >&2 sudo systemctl stop "$SERVICE" || rollback_rc=1 - restore_path "$profile_bin/teleo-kb" wrapper || rollback_rc=1 - restore_path "$profile_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 + restore_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1 + restore_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 + restore_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 + restore_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 - restore_path "$profile_bin/.livingip-runtime-revision" revision || rollback_rc=1 - sudo systemctl daemon-reload || rollback_rc=1 + restore_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 + restore_directory_state "$runtime_bin" runtime_bin || rollback_rc=1 + restore_directory_state "$runtime_parent" runtime_parent || rollback_rc=1 + restore_directory_state "$runtime_grandparent" runtime_grandparent || rollback_rc=1 + restore_directory_state "$dropin_dir" dropin_dir || rollback_rc=1 + if [ "$rollback_rc" -eq 0 ]; then + validate_restored_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1 + validate_restored_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 + validate_restored_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 + validate_restored_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 + validate_restored_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 + validate_restored_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 + validate_restored_directory "$runtime_grandparent" runtime_grandparent || rollback_rc=1 + validate_restored_directory "$runtime_parent" runtime_parent || rollback_rc=1 + validate_restored_directory "$runtime_bin" runtime_bin || rollback_rc=1 + validate_restored_directory "$dropin_dir" dropin_dir || rollback_rc=1 + fi + if [ "$rollback_rc" -eq 0 ]; then + sudo systemctl daemon-reload || rollback_rc=1 + fi + if [ "$rollback_rc" -eq 0 ]; then + assert_unit_configuration false || rollback_rc=1 + fi if [ "$rollback_rc" -eq 0 ]; then sudo systemctl start "$SERVICE" || rollback_rc=1 assert_service_running || rollback_rc=1 @@ -312,7 +723,13 @@ on_exit() { rc=1 fi fi - sudo rm -f -- "$dropin_stage" || true + sudo rm -f -- \ + "$wrapper_stage" \ + "$tool_stage" \ + "$permission_verifier_stage" \ + "$service_env_verifier_stage" \ + "$revision_stage" \ + "$dropin_stage" || true sudo rm -rf -- "$remote_tmp" || true exit "$rc" } @@ -321,36 +738,117 @@ trap 'on_exit $?' EXIT trap 'exit 130' INT trap 'exit 143' TERM -sudo rm -rf -- "$remote_tmp" -mkdir -p "$backup_dir" -tar -C "$remote_tmp" -xf - +sudo chmod 0711 "$remote_tmp" +sudo install -d -o root -g root -m 0755 "$payload_dir" +sudo install -d -o root -g root -m 0700 "$backup_dir" +sudo install -d -o teleo -g teleo -m 0700 "$work_dir" +sudo tar --no-same-owner --no-same-permissions --no-xattrs --no-acls --no-selinux -C "$payload_dir" -xf - +if sudo find "$payload_dir" -type l -print -quit | grep -q .; then + echo "Staged runtime payload contains a symlink." >&2 + exit 65 +fi +if sudo find "$payload_dir" ! -type d ! -type f -print -quit | grep -q .; then + echo "Staged runtime payload contains a non-regular entry." >&2 + exit 65 +fi +sudo chown -R root:root "$payload_dir" +sudo find "$payload_dir" -type d -exec chmod 0555 {} + +sudo find "$payload_dir" -type f -exec chmod 0444 {} + +sudo chmod 0555 \ + "$payload_dir/$WRAPPER_REL" \ + "$payload_dir/$TOOL_REL" \ + "$payload_dir/$PERMISSION_VERIFIER_REL" \ + "$payload_dir/$SERVICE_ENV_VERIFIER_REL" +test "$(sudo sha256sum "$payload_dir/$WRAPPER_REL" | awk '{print $1}')" = "$WRAPPER_SHA" +test "$(sudo sha256sum "$payload_dir/$TOOL_REL" | awk '{print $1}')" = "$TOOL_SHA" +test "$(sudo sha256sum "$payload_dir/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" +test "$(sudo sha256sum "$payload_dir/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" +test "$(sudo sha256sum "$payload_dir/$DROPIN_REL" | awk '{print $1}')" = "$DROPIN_SHA" +test "$(sudo stat -c '%U:%G:%a' "$remote_tmp")" = "root:root:711" +test "$(sudo stat -c '%U:%G:%a' "$payload_dir")" = "root:root:555" +test "$(sudo stat -c '%U:%G:%a' "$backup_dir")" = "root:root:700" +test "$(sudo stat -c '%U:%G:%a' "$work_dir")" = "teleo:teleo:700" +sudo -n -u teleo test ! -w "$remote_tmp" +sudo -n -u teleo test ! -w "$payload_dir" +sudo -n -u teleo test ! -w "$backup_dir" +sudo test -d "$runtime_anchor" +sudo test -d "$dropin_parent" +sudo test ! -L "$runtime_anchor" +sudo test ! -L "$dropin_parent" +test "$(sudo stat -c '%U:%G' "$runtime_anchor")" = "root:root" +test "$(sudo stat -c '%U:%G' "$dropin_parent")" = "root:root" +sudo -n -u teleo test ! -w "$runtime_anchor" +sudo -n -u teleo test ! -w "$dropin_parent" +for existing_directory in \ + "$runtime_grandparent" \ + "$runtime_parent" \ + "$runtime_bin" \ + "$dropin_dir"; do + assert_existing_root_directory "$existing_directory" +done +for existing_file in \ + "$runtime_bin/teleo-kb" \ + "$runtime_bin/cloudsql_memory_tool.py" \ + "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" \ + "$runtime_bin/verify_gcp_leoclean_service_environment.py" \ + "$runtime_bin/.livingip-runtime-revision" \ + "$dropin_dir/$DROPIN_NAME"; do + assert_existing_root_file "$existing_file" +done assert_service_running -run_scoped_status pre "$remote_tmp/$TOOL_REL" "$remote_tmp/gcloud-pre" +preflight_service_before="$(service_fingerprint)" +assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false +run_scoped_status pre "$payload_dir/$TOOL_REL" +assert_administrator_secret_denied +run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" +assert_service_fingerprint "$preflight_service_before" +if [ "$PREFLIGHT_ONLY" = true ]; then + systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager + exit 0 +fi -backup_path "$profile_bin/teleo-kb" wrapper -backup_path "$profile_bin/cloudsql_memory_tool.py" tool +backup_path "$runtime_bin/teleo-kb" wrapper +backup_path "$runtime_bin/cloudsql_memory_tool.py" tool +backup_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier +backup_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier backup_path "$dropin_dir/$DROPIN_NAME" dropin -backup_path "$profile_bin/.livingip-runtime-revision" revision +backup_path "$runtime_bin/.livingip-runtime-revision" revision +backup_directory_state "$runtime_parent" runtime_parent +backup_directory_state "$runtime_grandparent" runtime_grandparent +backup_directory_state "$runtime_bin" runtime_bin +backup_directory_state "$dropin_dir" dropin_dir mutation_started=true sudo systemctl stop "$SERVICE" -sudo install -d -o teleo -g teleo -m 0755 "$profile_bin" -sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$WRAPPER_REL" "$profile_bin/teleo-kb" -sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$TOOL_REL" "$profile_bin/cloudsql_memory_tool.py" +sudo install -d -o root -g root -m 0755 "$runtime_parent" +sudo install -d -o root -g root -m 0755 "$runtime_bin" +sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage" +sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage" +sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" "$permission_verifier_stage" +sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" "$service_env_verifier_stage" +sudo mv -f -- "$wrapper_stage" "$runtime_bin/teleo-kb" +sudo mv -f -- "$tool_stage" "$runtime_bin/cloudsql_memory_tool.py" +sudo mv -f -- "$permission_verifier_stage" "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" +sudo mv -f -- "$service_env_verifier_stage" "$runtime_bin/verify_gcp_leoclean_service_environment.py" sudo install -d -o root -g root -m 0755 "$dropin_dir" -sudo install -o root -g root -m 0644 "$remote_tmp/$DROPIN_REL" "$dropin_stage" +sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL" "$dropin_stage" sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME" -printf '%s\n' "$SOURCE_REVISION" | sudo tee "$profile_bin/.livingip-runtime-revision" >/dev/null -sudo chown teleo:teleo "$profile_bin/.livingip-runtime-revision" -sudo chmod 0644 "$profile_bin/.livingip-runtime-revision" +printf '%s\n' "$SOURCE_REVISION" | sudo tee "$revision_stage" >/dev/null +sudo chown root:root "$revision_stage" +sudo chmod 0644 "$revision_stage" +sudo mv -f -- "$revision_stage" "$runtime_bin/.livingip-runtime-revision" +assert_installed_files sudo systemctl daemon-reload +assert_unit_configuration sudo systemctl start "$SERVICE" assert_deployed_files assert_service_running -assert_runtime_environment -run_scoped_status post "$profile_bin/cloudsql_memory_tool.py" "$remote_tmp/gcloud-post" +assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py" +run_service_wrapper_status post +assert_administrator_secret_denied +run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" "$work_dir/permission-post/receipt.json" systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager mutation_started=false REMOTE @@ -358,4 +856,8 @@ REMOTE sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}" COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \ - "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command" + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command" diff --git a/docs/gcp-leoclean-runtime-reconciliation.md b/docs/gcp-leoclean-runtime-reconciliation.md index 754b15b..eafcb50 100644 --- a/docs/gcp-leoclean-runtime-reconciliation.md +++ b/docs/gcp-leoclean-runtime-reconciliation.md @@ -2,8 +2,9 @@ ## Scope -This runbook reconciles PR `#145` with the non-production GCP parallel Leo -service and removes its dependency on the PostgreSQL administrator credential. +This runbook reconciles PR `#148`, the least-privilege follow-up to PR `#145`, +with the non-production GCP parallel Leo service and removes its dependency on +the PostgreSQL administrator credential. It does not promote GCP, change the Telegram destination, copy newer VPS data, or make `teleo_canonical` production-authoritative. @@ -31,7 +32,8 @@ IAM or networking. commit. 2. `TELEO_KB_MODE=cloudsql`; missing tools or credentials fail closed. 3. Canonical zero-hit searches do not consult `teleo_restore` unless an - operator explicitly opts in. + operator explicitly opts in. The service is pinned to database + `teleo_canonical` with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`. 4. Leo can read the named canonical tables and stage only a `pending_review` proposal through `kb_stage.stage_leoclean_proposal(...)`. 5. Leo cannot directly insert, update, or delete `public.*` or @@ -55,7 +57,7 @@ required runtime secret has an equivalent secret-level binding. ### 2. Merge reviewed repository code -Run CI on the completed PR `#145` repair before deployment. The deployment +Run CI on the completed PR `#148` repair before deployment. The deployment source must be the resulting merged commit, not a working tree or unmerged branch. @@ -73,6 +75,9 @@ private VM path. Supply the new runtime password through `TELEO_LEOCLEAN_DB_PASSWORD`; the SQL file does not contain it. The migration: - creates or rotates `leoclean_kb_runtime`; +- keeps `leoclean_kb_runtime` `NOLOGIN` while both the `teleo_canonical` grant + phase and the legacy `teleo_kb` denial-verification phase run, and enables + `LOGIN` only in the final transaction after both phases pass; - creates a dedicated `NOLOGIN` function owner with no role memberships; - removes stale table, column, sequence, function, and role-membership grants; - grants exact canonical reads; @@ -94,26 +99,116 @@ by a `pg_catalog, pg_temp` search path, schema-qualified relations, a fixed The administrator password is used only for this bounded bootstrap. Retain no password output. +`teleo_kb` can still be connectable through its existing `PUBLIC CONNECT` +grant. That database-wide default is not treated as runtime authority. The +migration denies the scoped role access to the `teleo_restore` schema, the +runtime client is pinned to `teleo_canonical`, and audit fallback remains +disabled with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`. + ### 5. Preflight and deploy the merged runtime From the exact merged checkout, run: ```bash deploy/sync-gcp-leoclean-runtime.sh --dry-run +deploy/sync-gcp-leoclean-runtime.sh --preflight-only deploy/sync-gcp-leoclean-runtime.sh --restart ``` -The deploy script runs a scoped, redacted database status preflight before it -installs files or restarts the service. It refuses dirty or unmerged runtime -files and refuses a live install without `--restart`. After preflight it stops -the parallel service, atomically replaces the existing Cloud SQL systemd -drop-in, installs the reviewed wrapper and helper, records the Git revision, -starts the service, and verifies all three file hashes. A failure after mutation -restores the prior wrapper, helper, drop-in, and revision as one set before the -old service is restarted. +`--preflight-only` extracts the candidate helper and permission verifier into a +root-created, root-owned, non-writable VM payload directory, verifies the actual +service `MainPID` environment, runs the candidate private Cloud SQL status path, +and runs the sanitized positive-and-negative permission verifier. A separate +`teleo`-owned directory holds only sanitized receipts; it never feeds an install +or rollback. The preflight does not stop or restart the service and does not +install or replace runtime files. It +captures `ActiveState`, `SubState`, `MainPID`, and `NRestarts` before and after +the checks and fails if any value changes. The preflight must pass before +`--restart` is attempted. + +The deploy script refuses dirty, symlinked, or unmerged runtime files and +refuses a live install without `--restart`. Its payload and rollback backup are +root-owned and non-writable by `teleo`; existing runtime intermediates and +targets must be real root-owned, non-writable paths. After preflight it stops +the parallel service, atomically replaces the existing Cloud SQL systemd drop-in, installs the +reviewed wrapper, helper, and permission verifier into the root-controlled +`/usr/local/libexec/livingip/leoclean-kb` directory, records the Git revision, +and verifies every installed hash, owner, mode, path type, and revision before +systemd reloads the unit or starts the service. The wrapper uses fixed +`/bin/bash` and `/usr/bin/python3` interpreters, and every directory on the +service `PATH` must be root-owned and non-writable by `teleo`. Every ancestor of that +directory, the directory itself, the wrapper, helper, verifier, and revision +marker must be non-writable by `teleo`. The systemd service receives that +directory as a read-only bind at its existing +`/home/teleo/.hermes/profiles/leoclean/bin` path; verification compares the +source and service-namespace inode and hashes and proves the mount is read-only. +To prevent the `teleo` service user from renaming a writable ancestor and +recreating that absolute path, the service namespace mounts +`/home/teleo/.hermes` read-only, allows writes back only to the profile's +`state` and `workspace` subdirectories, and removes `CAP_SYS_ADMIN`. Live +verification requires `.hermes` and `bin` to be read-only, both allowed +subdirectories to be writable, `/home` to be non-writable by `teleo`, and the +effective capability set to exclude `CAP_SYS_ADMIN`. + +Preflight, post-restart, and `--verify-only` checks inspect the environment of the actual +systemd `MainPID` through `/proc//environ`, rather than trusting the +configured unit environment alone. A standalone fail-closed verifier requires +the exact reviewed private target and runtime role, requires +`TELEO_GCP_METADATA_ONLY=1`, and rejects all PostgreSQL, Cloud SDK, proxy, TLS +trust, broader Google credential, shell-startup, dynamic-loader, Python-loader, +and administrator-secret overrides without printing their values. Effective +`EnvironmentFile` use and drop-ins ordered after the reviewed configuration are +rejected before restart. This blocks +pre-wrapper injection through fields such as `BASH_ENV`, any `LD_*`, +`PYTHONPATH`, `PYTHONHTTPSVERIFY`, or `SSLKEYLOGFILE`. The checks also require +the reviewed drop-in to appear in `DropInPaths`. The wrapper status probe then +imports the validated environment from the actual `MainPID` with `nsenter +--env`, enters its mount namespace, drops to `teleo`, and invokes the bound +wrapper without reinjecting any database identity setting. + +The service runtime bypasses Cloud SDK configuration and credential caches +entirely. The helper obtains the attached service-account email and a bounded +access token from the fixed GCE metadata endpoint, requires the exact expected +service account, and accesses only the named scoped Secret Manager version over +the fixed HTTPS API. Its URL opener disables proxies. The scoped status read +and administrator-secret IAM denial are rerun after restart. The denial probe +imports the actual systemd `MainPID` environment and mount namespace, drops to +the service's `teleo` identity, and must observe the exact +`secretmanager.versions.access` permission denial. No token or +secret value is retained. The status read executes the bound `teleo-kb` wrapper +from inside the service mount namespace, proving the wrapper and its adjacent +helper rather than bypassing them with a direct helper invocation. A failure +after mutation restores the prior runtime/drop-in directory existence and +metadata, wrapper, helper, both verifiers, drop-in, and revision as one set, +validates it against the root-only backup, and only then reloads systemd and +restarts the old service. ### 6. Verify positive and negative behavior +`--preflight-only`, `--restart`, and `--verify-only` run the verifier as the +`teleo` Unix user. To rerun the installed verifier directly on the VM and +retain its sanitized JSON receipt, use: + +```bash +receipt_dir="$(mktemp -d /tmp/leoclean-permission-receipt.XXXXXX)" +sudo chown teleo:teleo "$receipt_dir" +sudo chmod 0700 "$receipt_dir" +run_id="$(date -u +%Y%m%d%H%M%S)-manual" +sudo -n -u teleo env -i HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + python3 /usr/local/libexec/livingip/leoclean-kb/verify_gcp_leoclean_runtime_permissions.py \ + --run-id "$run_id" \ + --output "$receipt_dir/permission-receipt.json" +sudo stat -c '%U:%G:%a' "$receipt_dir/permission-receipt.json" +``` + +The command must exit zero, print the same canonical JSON that it writes, and +leave the receipt as `teleo:teleo:600`. A passing receipt must report +`status=pass`, `mode=live_private_gcp_staging`, and both `required_tier` and +`current_tier` as `T3_live_readonly`. Those values are a receipt contract, not +a claim that this runbook has already been exercised successfully on the live +staging VM. + Retain a redacted receipt containing: - merged Git commit and deployed file hashes; @@ -122,13 +217,28 @@ Retain a redacted receipt containing: configuration; - `current_database()` and `current_user` showing `teleo_canonical|leoclean_kb_runtime`; +- private server address `10.61.0.3`, port `5432`, and an active SSL session; +- exact safe runtime-role attributes and zero membership edges in + `pg_auth_members`; +- zero effective database/schema `CREATE`, persistent ownership, table/column + DML, sequence privileges, SELECT outside the exact table allowlist, or + executable unexpected `SECURITY DEFINER` functions, plus the exact staging + function owner/search-path/ACL contract; - a real canonical status/search receipt; - a transaction-rolled-back call to `stage_leoclean_proposal(...)` that returns - `pending_review`; -- denied direct insert into `kb_stage.kb_proposals`; -- denied canonical `public.*` write; -- denied reviewer/apply security-definer function; -- denied forged proposer identity and denied `SET ROLE` escalation; + `pending_review` for canonical proposer `leo`, with zero matching canary rows + both before and after the transaction; +- denied direct insert, update, and delete on `kb_stage.kb_proposals`; +- denied insert, update, and delete on canonical `public.claims`; +- denied reviewer/apply security-definer functions; +- exact function-catalog posture: only the five-argument staging function is + executable, the forged six-argument overload is absent, and each expected + reviewer/apply function exists but is not executable by the runtime role; +- unavailable forged-proposer overload and denied `SET ROLE` escalation into + the staging owner, reviewer, apply, or administrator roles; +- denied access to `teleo_restore` even when `teleo_kb` remains connectable; +- readable scoped runtime secret and an IAM permission denial for the + administrator secret, with neither secret value retained; - zero Telegram messages and zero committed canary rows. ### 7. Remove administrator-secret access diff --git a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py index dd2edbe..3367207 100755 --- a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py +++ b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py @@ -1,20 +1,24 @@ #!/usr/bin/env python3 """Cloud SQL memory and KB-proposal bridge for the GCP leoclean profile. -Canonical claim/strategy application remains review-gated. Read commands query -the canonical Teleo KB first, then fall back to the restored `teleo_restore` -shadow schema in Cloud SQL. Proposal commands write staged review records into -`kb_stage.kb_proposals`; they do not apply canonical truth directly. +Canonical claim/strategy application remains review-gated. The production-like +Leo runtime is pinned to the private canonical Cloud SQL database and never +falls back to the restored `teleo_restore` shadow schema. Proposal commands +write staged review records into `kb_stage.kb_proposals`; they do not apply +canonical truth directly. """ from __future__ import annotations import argparse +import base64 +import binascii import hashlib import json import os import re import subprocess +import urllib.request from typing import Any STOPWORDS = { @@ -76,6 +80,38 @@ DEFAULT_CLAIM_BASE_URL = "https://leo.livingip.xyz" RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1" RUNTIME_DB_USER = "leoclean_kb_runtime" RUNTIME_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" +RUNTIME_GCP_PROJECT = "teleo-501523" +RUNTIME_CLOUDSQL_HOST = "10.61.0.3" +RUNTIME_CLOUDSQL_PORT = "5432" +RUNTIME_CLOUDSQL_DB = "teleo_canonical" +RUNTIME_SERVICE_ACCOUNT = "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" +GCE_METADATA_EMAIL_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" +GCE_METADATA_TOKEN_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" +RUNTIME_SECRET_ACCESS_URL = ( + "https://secretmanager.googleapis.com/v1/projects/teleo-501523/secrets/" + "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password/versions/latest:access" +) +METADATA_HEADERS = {"Metadata-Flavor": "Google"} +HTTP_TIMEOUT_SECONDS = 3.0 +MAX_METADATA_EMAIL_BYTES = 256 +MAX_METADATA_TOKEN_BYTES = 4096 +MAX_SECRET_RESPONSE_BYTES = 8192 +MAX_RUNTIME_SECRET_BYTES = 4096 +RUNTIME_TLS_TRUST_ENVIRONMENT = frozenset( + { + "CURL_CA_BUNDLE", + "PYTHONHTTPSVERIFY", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SSLKEYLOGFILE", + } +) +RUNTIME_PSQL_ENV_BASE = { + "PATH": "/usr/bin:/bin", + "PGSSLMODE": "require", + "PGCONNECT_TIMEOUT": "8", +} CLONE_READ_ONLY_PGOPTIONS = "-c default_transaction_read_only=on" RECEIPTED_READ_COMMANDS = frozenset( { @@ -93,6 +129,18 @@ RECEIPTED_READ_COMMANDS = frozenset( ) +class _RejectRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *_args: Any, **_kwargs: Any) -> None: + return None + + +RUNTIME_PROXY_HANDLER = urllib.request.ProxyHandler({}) +RUNTIME_HTTP_OPENER = urllib.request.build_opener( + RUNTIME_PROXY_HANDLER, + _RejectRedirectHandler(), +) + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--host", default=os.environ.get("TELEO_CLOUDSQL_HOST", "10.61.0.3")) @@ -246,8 +294,47 @@ def validate_runtime_connection(args: argparse.Namespace) -> None: raise SystemExit(f"Leo runtime requires database user {RUNTIME_DB_USER}.") if password_secret != RUNTIME_PASSWORD_SECRET: raise SystemExit(f"Leo runtime requires scoped password secret {RUNTIME_PASSWORD_SECRET}.") - if "PGPASSWORD" in os.environ: - raise SystemExit("Refusing inherited PGPASSWORD in Leo runtime mode.") + expected_fields = { + "project": RUNTIME_GCP_PROJECT, + "host": RUNTIME_CLOUDSQL_HOST, + "port": RUNTIME_CLOUDSQL_PORT, + "db": RUNTIME_CLOUDSQL_DB, + "canonical_db": RUNTIME_CLOUDSQL_DB, + } + for field, expected in expected_fields.items(): + actual = str(getattr(args, field, "") or "") + if actual != expected: + raise SystemExit(f"Leo runtime requires {field}={expected}.") + if os.environ.get("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") != "0": + raise SystemExit("Leo runtime requires TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0.") + if os.environ.get("TELEO_GCP_METADATA_ONLY") != "1": + raise SystemExit("Leo runtime requires TELEO_GCP_METADATA_ONLY=1.") + forbidden_environment = any( + normalized_key.startswith("PG") + or normalized_key.startswith("CLOUDSDK_") + or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS" + or normalized_key.startswith("LD_") + or normalized_key.startswith("BASH_FUNC_") + or (normalized_key.startswith("PYTHON") and normalized_key != "PYTHONUNBUFFERED") + or normalized_key in { + "BASH_ENV", + "BASHOPTS", + "CDPATH", + "ENV", + "GLOBIGNORE", + "IFS", + "PS4", + "SHELLOPTS", + } + or normalized_key == "PROXY" + or normalized_key.endswith("_PROXY") + or normalized_key in RUNTIME_TLS_TRUST_ENVIRONMENT + for normalized_key in (key.upper() for key in os.environ) + ) + if forbidden_environment: + raise SystemExit( + "Refusing inherited credential, proxy, TLS, or PostgreSQL environment in Leo runtime mode." + ) return if mode != "clone-readonly": @@ -261,8 +348,8 @@ def validate_runtime_connection(args: argparse.Namespace) -> None: raise SystemExit("Clone credential mode permits read commands only.") if pgoptions != CLONE_READ_ONLY_PGOPTIONS: raise SystemExit("Clone credential mode requires server-enforced default_transaction_read_only=on.") - if not user or (not password_secret and not os.environ.get("PGPASSWORD")): - raise SystemExit("Clone credential mode requires an explicit user and credential source.") + if not user or not os.environ.get("PGPASSWORD"): + raise SystemExit("Clone credential mode requires an explicit user and inherited test credential.") def terms_for(query: str) -> list[str]: @@ -305,48 +392,138 @@ def sql_array(values: list[str], cast: str = "text") -> str: return "array[" + ",".join(sql_literal(value) for value in values) + f"]::{cast}[]" +def _read_http_response(request: urllib.request.Request, *, max_bytes: int, purpose: str) -> bytes: + try: + with RUNTIME_HTTP_OPENER.open(request, timeout=HTTP_TIMEOUT_SECONDS) as response: + if response.getcode() != 200: + raise SystemExit(f"{purpose} request failed.") + body = response.read(max_bytes + 1) + except SystemExit: + raise + # This is a credential-bearing boundary. Never propagate provider, proxy, + # TLS, or test-double exception text because it can contain bearer data. + except Exception: + raise SystemExit(f"{purpose} request failed.") from None + if not isinstance(body, bytes) or len(body) > max_bytes: + raise SystemExit(f"{purpose} response was invalid.") + return body + + +def _strict_json_object(body: bytes, *, purpose: str) -> dict[str, Any]: + def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError("duplicate JSON key") + value[key] = item + return value + + try: + decoded = body.decode("utf-8", errors="strict") + value = json.loads( + decoded, + object_pairs_hook=reject_duplicate_keys, + parse_constant=lambda _constant: (_ for _ in ()).throw(ValueError("non-finite JSON value")), + ) + except (UnicodeDecodeError, ValueError, TypeError): + raise SystemExit(f"{purpose} response was invalid.") from None + if not isinstance(value, dict): + raise SystemExit(f"{purpose} response was invalid.") + return value + + +def _metadata_request(url: str, *, max_bytes: int, purpose: str) -> bytes: + request = urllib.request.Request(url, headers=METADATA_HEADERS, method="GET") + return _read_http_response(request, max_bytes=max_bytes, purpose=purpose) + + +def runtime_password_from_metadata() -> str: + email_body = _metadata_request( + GCE_METADATA_EMAIL_URL, + max_bytes=MAX_METADATA_EMAIL_BYTES, + purpose="Runtime metadata identity", + ) + try: + service_account = email_body.decode("ascii", errors="strict") + except UnicodeDecodeError: + raise SystemExit("Runtime metadata identity response was invalid.") from None + if service_account != RUNTIME_SERVICE_ACCOUNT: + raise SystemExit("Runtime metadata identity did not match the required service account.") + + token_value = _strict_json_object( + _metadata_request( + GCE_METADATA_TOKEN_URL, + max_bytes=MAX_METADATA_TOKEN_BYTES, + purpose="Runtime metadata token", + ), + purpose="Runtime metadata token", + ) + access_token = token_value.get("access_token") + token_type = token_value.get("token_type") + expires_in = token_value.get("expires_in") + if ( + token_type != "Bearer" + or not isinstance(access_token, str) + or not re.fullmatch(r"[!-~]{1,4096}", access_token) + or not isinstance(expires_in, int) + or isinstance(expires_in, bool) + or expires_in <= 0 + ): + raise SystemExit("Runtime metadata token response was invalid.") + + secret_request = urllib.request.Request( + RUNTIME_SECRET_ACCESS_URL, + headers={"Accept": "application/json", "Authorization": f"Bearer {access_token}"}, + method="GET", + ) + secret_value = _strict_json_object( + _read_http_response( + secret_request, + max_bytes=MAX_SECRET_RESPONSE_BYTES, + purpose="Scoped runtime secret access", + ), + purpose="Scoped runtime secret access", + ) + payload = secret_value.get("payload") + encoded_secret = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(encoded_secret, str): + raise SystemExit("Scoped runtime secret response was invalid.") + try: + credential_bytes = base64.b64decode(encoded_secret, validate=True) + except (binascii.Error, ValueError): + raise SystemExit("Scoped runtime secret response was invalid.") from None + if not credential_bytes or len(credential_bytes) > MAX_RUNTIME_SECRET_BYTES: + raise SystemExit("Scoped runtime secret response was invalid.") + if any(byte in credential_bytes for byte in (0, 10, 13)): + raise SystemExit("Scoped runtime secret response was invalid.") + try: + return credential_bytes.decode("utf-8", errors="strict") + except UnicodeDecodeError: + raise SystemExit("Scoped runtime secret response was invalid.") from None + + def password(args: argparse.Namespace) -> str: validate_runtime_connection(args) - if args.credential_mode == "clone-readonly" and os.environ.get("PGPASSWORD"): + if args.credential_mode == "runtime": + return runtime_password_from_metadata() + if os.environ.get("PGPASSWORD"): return os.environ["PGPASSWORD"] - if not args.password_secret: - raise SystemExit("Scoped Cloud SQL password secret is not configured.") - result = subprocess.run( - [ - "gcloud", - "secrets", - "versions", - "access", - "latest", - f"--secret={args.password_secret}", - f"--project={args.project}", - ], - text=True, - capture_output=True, - check=False, - ) - if result.returncode != 0: - raise SystemExit(f"Secret access failed: {result.stderr.strip()}") - credential = result.stdout.strip() - if not credential: - raise SystemExit("Scoped Cloud SQL password secret resolved to an empty value.") - return credential + raise SystemExit("Clone credential mode requires an inherited test credential.") def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: - env = os.environ.copy() - for key in tuple(env): - if key.startswith("PG"): - env.pop(key, None) + credential = password(args) + if args.credential_mode == "runtime": + env = dict(RUNTIME_PSQL_ENV_BASE) + else: + env = {key: value for key, value in os.environ.items() if not key.startswith("PG")} env.update( { "PGHOST": str(args.host), "PGPORT": str(args.port), "PGDATABASE": str(db or args.db), "PGUSER": str(args.user), - "PGSSLMODE": "require", - "PGCONNECT_TIMEOUT": "8", - "PGPASSWORD": password(args), + "PGPASSWORD": credential, } ) if args.credential_mode == "clone-readonly": @@ -360,9 +537,7 @@ def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: check=False, ) if result.returncode != 0: - raise SystemExit( - f"Cloud SQL query failed ({result.returncode})\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" - ) + raise SystemExit(f"Cloud SQL query failed with exit status {result.returncode}.") return result.stdout @@ -385,6 +560,10 @@ def database_read_marker(args: argparse.Namespace) -> dict[str, Any]: select jsonb_build_object( 'database', current_database(), 'database_user', current_user, + 'server_addr', inet_server_addr()::text, + 'server_port', inet_server_port(), + 'ssl', coalesce((select ssl from pg_catalog.pg_stat_ssl where pid = pg_backend_pid()), false), + 'ssl_version', (select version from pg_catalog.pg_stat_ssl where pid = pg_backend_pid()), 'system_identifier', (select system_identifier::text from pg_control_system()), 'wal_lsn', pg_current_wal_lsn()::text )::text; @@ -392,12 +571,34 @@ select jsonb_build_object( rows = psql_json_lines(args, sql, db=args.canonical_db) if len(rows) != 1: raise SystemExit("Canonical database read marker returned an invalid row count.") - return rows[0] + marker = rows[0] + expected_database = str(args.canonical_db or "") + expected_user = str(args.user or "") + if marker.get("database") != expected_database or marker.get("database_user") != expected_user: + raise SystemExit("Canonical database read marker returned an unexpected database identity.") + if getattr(args, "credential_mode", "runtime") == "runtime": + if marker.get("server_addr") != RUNTIME_CLOUDSQL_HOST: + raise SystemExit("Canonical database read marker returned an unexpected server address.") + if str(marker.get("server_port")) != RUNTIME_CLOUDSQL_PORT: + raise SystemExit("Canonical database read marker returned an unexpected server port.") + if marker.get("ssl") is not True: + raise SystemExit("Canonical database read marker did not prove an encrypted connection.") + return marker def _read_marker_stable(before: dict[str, Any], after: dict[str, Any]) -> bool: return all( - before.get(key) == after.get(key) for key in ("database", "database_user", "system_identifier", "wal_lsn") + before.get(key) == after.get(key) + for key in ( + "database", + "database_user", + "server_addr", + "server_port", + "ssl", + "ssl_version", + "system_identifier", + "wal_lsn", + ) ) @@ -470,6 +671,10 @@ def build_retrieval_receipt( "attempts": attempts, "database": before.get("database"), "database_user": before.get("database_user"), + "server_addr": before.get("server_addr"), + "server_port": before.get("server_port"), + "ssl": before.get("ssl"), + "ssl_version": before.get("ssl_version"), "system_identifier": before.get("system_identifier"), "wal_lsn_before": before.get("wal_lsn"), "wal_lsn_after": after.get("wal_lsn"), diff --git a/hermes-agent/leoclean-bin/teleo-kb b/hermes-agent/leoclean-bin/teleo-kb index 369f09b..d5b3e15 100755 --- a/hermes-agent/leoclean-bin/teleo-kb +++ b/hermes-agent/leoclean-bin/teleo-kb @@ -1,9 +1,15 @@ -#!/usr/bin/env bash +#!/bin/bash set -euo pipefail PROFILE_DIR=${HERMES_HOME:-/home/teleo/.hermes/profiles/leoclean} KB_TOOL="$PROFILE_DIR/bin/kb_tool.py" -CLOUDSQL_TOOL="$PROFILE_DIR/bin/cloudsql_memory_tool.py" +SCRIPT_PATH=${BASH_SOURCE[0]} +SCRIPT_DIR=${SCRIPT_PATH%/*} +if [ "$SCRIPT_DIR" = "$SCRIPT_PATH" ]; then + SCRIPT_DIR=. +fi +SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)" +CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py" MODE=${TELEO_KB_MODE:-auto} COMMAND=${1:-} @@ -31,17 +37,22 @@ run_cloudsql() { echo "teleo-kb: Cloud SQL tool is missing or not executable: $CLOUDSQL_TOOL" >&2 exit 69 fi - for dependency in python3 psql gcloud; do - if ! command -v "$dependency" >/dev/null 2>&1; then - echo "teleo-kb: required Cloud SQL dependency is unavailable: $dependency" >&2 - exit 69 - fi - done + if [ ! -x /usr/bin/python3 ]; then + echo "teleo-kb: required Cloud SQL dependency is unavailable: /usr/bin/python3" >&2 + exit 69 + fi + if ! command -v psql >/dev/null 2>&1; then + echo "teleo-kb: required Cloud SQL dependency is unavailable: psql" >&2 + exit 69 + fi extra_args=() while IFS= read -r arg; do [ -n "$arg" ] && extra_args+=("$arg") done < <(cloudsql_args) - exec python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}" + if [ "${#extra_args[@]}" -gt 0 ]; then + exec /usr/bin/python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}" + fi + exec /usr/bin/python3 "$CLOUDSQL_TOOL" "$@" } case "$MODE" in @@ -49,10 +60,10 @@ case "$MODE" in run_cloudsql "$@" ;; local) - exec python3 "$KB_TOOL" --local "$@" + exec /usr/bin/python3 "$KB_TOOL" --local "$@" ;; remote) - exec python3 "$KB_TOOL" "$@" + exec /usr/bin/python3 "$KB_TOOL" "$@" ;; auto) # Canonical commands must fail closed on Cloud SQL errors. Falling through @@ -62,9 +73,9 @@ case "$MODE" in run_cloudsql "$@" fi if command -v docker >/dev/null 2>&1; then - exec python3 "$KB_TOOL" --local "$@" + exec /usr/bin/python3 "$KB_TOOL" --local "$@" fi - exec python3 "$KB_TOOL" "$@" + exec /usr/bin/python3 "$KB_TOOL" "$@" ;; *) echo "Unsupported TELEO_KB_MODE=$MODE; expected auto, cloudsql, local, or remote" >&2 diff --git a/ops/gcp_leoclean_runtime_role.sql b/ops/gcp_leoclean_runtime_role.sql index cd090a3..da884c8 100644 --- a/ops/gcp_leoclean_runtime_role.sql +++ b/ops/gcp_leoclean_runtime_role.sql @@ -1,5 +1,4 @@ \set ON_ERROR_STOP on -\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD -- One-time GCP staging provisioning for Leo's Cloud SQL identity. The password -- is supplied only through the process environment and is never stored here. @@ -8,7 +7,10 @@ -- through one narrowly-scoped SECURITY DEFINER function. It cannot inherit or -- SET ROLE into another principal, forge the proposer identity, write directly -- to canonical/staging tables, or inspect the legacy teleo_restore audit schema. -select 'create role leoclean_kb_runtime login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8' +-- Keep the runtime unable to authenticate until both database phases have +-- completed and verified. Any failure before the final transaction therefore +-- leaves a disabled role instead of a partially provisioned login. +select 'create role leoclean_kb_runtime nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8' where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_runtime') \gexec @@ -16,15 +18,19 @@ select 'create role leoclean_kb_stage_owner nologin nosuperuser nocreatedb nocre where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_stage_owner') \gexec -select format('alter role leoclean_kb_runtime password %L', :'runtime_password') -\gexec - alter role leoclean_kb_runtime - with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; + with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; alter role leoclean_kb_stage_owner with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1 password null; +-- Rotate the credential only after an idempotent rerun has disabled any +-- pre-existing LOGIN role. This ordering has an intentional autocommit +-- boundary: interruption can leave NOLOGIN, never a still-enabled old role. +\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD +select format('alter role leoclean_kb_runtime password %L', :'runtime_password') +\gexec + alter role leoclean_kb_runtime reset all; alter role leoclean_kb_stage_owner reset all; @@ -261,7 +267,7 @@ begin select 1 from pg_catalog.pg_roles role_row where role_row.rolname = 'leoclean_kb_runtime' - and (not role_row.rolcanlogin + and (role_row.rolcanlogin or role_row.rolsuper or role_row.rolcreatedb or role_row.rolcreaterole @@ -270,7 +276,7 @@ begin or role_row.rolbypassrls or role_row.rolconnlimit <> 8) ) then - raise exception 'leoclean_kb_runtime has unsafe role attributes'; + raise exception 'leoclean_kb_runtime has unsafe provisioning role attributes'; end if; if exists ( @@ -818,6 +824,36 @@ commit; \connect teleo_canonical +-- LOGIN is the last state change. If this verification fails, the transaction +-- rolls back and the role remains NOLOGIN while prior least-privilege grants +-- stay inert for direct authentication. +begin; + +alter role leoclean_kb_runtime + with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; + +do $login_verification$ +begin + if exists ( + select 1 + from pg_catalog.pg_roles role_row + where role_row.rolname = 'leoclean_kb_runtime' + and (not role_row.rolcanlogin + or role_row.rolsuper + or role_row.rolcreatedb + or role_row.rolcreaterole + or role_row.rolinherit + or role_row.rolreplication + or role_row.rolbypassrls + or role_row.rolconnlimit <> 8) + ) then + raise exception 'leoclean_kb_runtime final login attributes are unsafe'; + end if; +end +$login_verification$; + +commit; + select jsonb_build_object( 'role', rolname, 'superuser', rolsuper, @@ -835,6 +871,7 @@ select jsonb_build_object( 'can_select_claims', has_table_privilege(rolname, 'public.claims', 'SELECT'), 'can_select_proposals', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'SELECT'), 'can_insert_proposals_directly', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'INSERT'), + 'can_connect_legacy_database', has_database_privilege(rolname, 'teleo_kb', 'CONNECT'), 'can_create_database_objects', has_database_privilege(rolname, 'teleo_canonical', 'CREATE'), 'can_create_temp_objects', has_database_privilege(rolname, 'teleo_canonical', 'TEMP'), 'can_stage_proposal', has_function_privilege( diff --git a/ops/verify_gcp_leoclean_runtime_permissions.py b/ops/verify_gcp_leoclean_runtime_permissions.py new file mode 100644 index 0000000..bdd44dd --- /dev/null +++ b/ops/verify_gcp_leoclean_runtime_permissions.py @@ -0,0 +1,1838 @@ +#!/usr/bin/env python3 +"""Retain a sanitized least-privilege receipt for the GCP Leo runtime role. + +This verifier is intentionally service-independent. It must run on the private +GCP VM as ``teleo`` and connects directly to the private Cloud SQL address. The +only database credential is read from the scoped Secret Manager secret into +process memory, passed to each ``psql`` child through ``PGPASSWORD``, and never +included in an argument, message, receipt, or output file. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pwd +import re +import stat +import subprocess +import sys +import tempfile +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +PROJECT_ID = "teleo-501523" +PRIVATE_CLOUDSQL_HOST = "10.61.0.3" +PRIVATE_CLOUDSQL_PORT = 5432 +CANONICAL_DATABASE = "teleo_canonical" +LEGACY_DATABASE = "teleo_kb" +RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime" +STAGE_OWNER_DATABASE_ROLE = "leoclean_kb_stage_owner" +RUNTIME_UNIX_USER = "teleo" +SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" +ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password" +GCLOUD_BIN = "/usr/bin/gcloud" +PSQL_BIN = "/usr/bin/psql" +COMMAND_TIMEOUT_SECONDS = 30 +CHILD_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +RUN_ID_RE = re.compile(r"[a-z0-9][a-z0-9-]{6,62}[a-z0-9]\Z") +SQLSTATE_RE = re.compile(r"(?m)^(?:ERROR|FATAL):\s+([0-9A-Z]{5}):") +IAM_PERMISSION = "secretmanager.versions.access" +REQUIRED_TIER = "T3_live_readonly" +LOWER_TIER = "T2_runtime" +ARTIFACT_NAME = "gcp_leoclean_runtime_permissions" +STAGE_FUNCTION_SIGNATURE = "kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)" + +# PostgreSQL stores the exact dollar-quoted function body in pg_proc.prosrc. +# A source-parity regression below binds this constant to the provisioning SQL, +# while the live verifier compares the catalog body without retaining it. +EXPECTED_STAGE_FUNCTION_SOURCE = """ +declare + staged kb_stage.kb_proposals%rowtype; + proposer_id uuid; +begin + if session_user <> 'leoclean_kb_runtime' then + raise exception 'stage_leoclean_proposal is restricted to leoclean_kb_runtime' + using errcode = '42501'; + end if; + if p_proposal_type not in ('revise_claim', 'revise_strategy', 'add_edge') then + raise exception 'unsupported Leo proposal type: %', p_proposal_type using errcode = '22023'; + end if; + if nullif(btrim(p_source_ref), '') is null then + raise exception 'source_ref is required' using errcode = '22023'; + end if; + if nullif(btrim(p_rationale), '') is null then + raise exception 'rationale is required' using errcode = '22023'; + end if; + if p_payload is null or jsonb_typeof(p_payload) <> 'object' then + raise exception 'proposal payload must be a JSON object' using errcode = '22023'; + end if; + + select agent.id + into proposer_id + from public.agents as agent + where agent.handle = 'leo'; + + if proposer_id is null then + raise exception 'canonical Leo agent row is required' using errcode = '23503'; + end if; + + insert into kb_stage.kb_proposals ( + proposal_type, + status, + proposed_by_handle, + proposed_by_agent_id, + channel, + source_ref, + rationale, + payload + ) values ( + p_proposal_type, + 'pending_review', + 'leo', + proposer_id, + coalesce(nullif(p_channel, ''), 'telegram'), + p_source_ref, + p_rationale, + p_payload + ) + returning * into staged; + + return to_jsonb(staged); +end +""" +EXPECTED_STAGE_FUNCTION_SOURCE_SHA256 = "89f78d5ad7bc7227a4588baf42f7f77e2162b3c4647a250f7c88716b0debfdea" +MAX_STAGE_FUNCTION_SOURCE_BYTES = 65_536 + +STAGE_FUNCTION_METADATA_EXPECTATIONS: tuple[tuple[str, Any], ...] = ( + ("argument_defaults", 0), + ( + "argument_names", + ["p_proposal_type", "p_channel", "p_source_ref", "p_rationale", "p_payload"], + ), + ("argument_types", ["text", "text", "text", "text", "jsonb"]), + ("kind", "f"), + ("language", "plpgsql"), + ("leakproof", False), + ("owner", STAGE_OWNER_DATABASE_ROLE), + ("parallel", "u"), + ("returns_set", False), + ("result", "jsonb"), + ("security_definer", True), + ("strict", False), + ("volatility", "v"), +) + +STAGE_OWNER_INSERT_COLUMNS: tuple[str, ...] = ( + "proposal_type", + "status", + "proposed_by_handle", + "proposed_by_agent_id", + "channel", + "source_ref", + "rationale", + "payload", +) + +FUNCTION_PRIVILEGE_EXPECTATIONS: tuple[tuple[str, str, bool, bool], ...] = ( + ( + "stage_leoclean_proposal_5_arg", + "kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)", + True, + True, + ), + ( + "stage_leoclean_proposal_6_arg", + "kb_stage.stage_leoclean_proposal(text,text,text,text,text,jsonb)", + False, + False, + ), + ( + "approve_strict_proposal", + "kb_stage.approve_strict_proposal(uuid,text,jsonb,text,text)", + True, + False, + ), + ( + "assert_approved_proposal", + "kb_stage.assert_approved_proposal(uuid,text,jsonb,text,uuid,timestamptz,text)", + True, + False, + ), + ( + "finish_approved_proposal", + "kb_stage.finish_approved_proposal(uuid,text,jsonb,text,uuid,timestamptz,text,text)", + True, + False, + ), +) + +READ_TABLE_ALLOWLIST: tuple[tuple[str, str], ...] = ( + ("public", "agents"), + ("public", "claims"), + ("public", "claim_evidence"), + ("public", "claim_edges"), + ("public", "sources"), + ("public", "personas"), + ("public", "strategies"), + ("public", "beliefs"), + ("public", "blindspots"), + ("public", "agent_roles"), + ("public", "peer_models"), + ("public", "behavioral_rules"), + ("public", "contributor_rules"), + ("public", "reasoning_tools"), + ("public", "governance_gates"), + ("kb_stage", "kb_proposals"), +) + +CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( + "column_dml_grants", + "database_create_grants", + "missing_allowed_table_selects", + "owned_database_objects", + "owner_database_create_grants", + "owner_direct_database_acl_entries", + "owner_missing_agent_select_columns", + "owner_missing_proposal_insert_columns", + "owner_schema_create_grants", + "owner_sequence_grants", + "owner_table_dml_grants", + "owner_unexpected_column_dml_grants", + "owner_unexpected_column_selects", + "owner_unexpected_owned_database_objects", + "owner_unexpected_routine_execute", + "owner_unexpected_schema_usage", + "owner_unexpected_table_selects", + "schema_create_grants", + "sequence_grants", + "table_dml_grants", + "unexpected_column_selects", + "unexpected_direct_database_acl_entries", + "unexpected_routine_execute", + "unexpected_schema_usage", + "unexpected_security_definer_execute", + "unexpected_table_selects", +) + +CATALOG_TRUE_FIELDS: tuple[str, ...] = ( + "canonical_connect", + "owner_grant_contract", + "owner_schema_usage", + "runtime_connect_acl_exact", + "runtime_schema_usage", +) + +LEGACY_CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( + "owner_column_privileges", + "owner_routine_privileges", + "owner_schema_privileges", + "owner_sequence_privileges", + "owner_table_privileges", + "runtime_column_privileges", + "runtime_routine_privileges", + "runtime_schema_privileges", + "runtime_sequence_privileges", + "runtime_table_privileges", +) + + +@dataclass(frozen=True) +class CommandResult: + returncode: int + stdout: str + stderr: str + + +Runner = Callable[[list[str], Mapping[str, str], int, bool], CommandResult] + + +@dataclass(frozen=True) +class NegativeCheck: + name: str + database: str + sql: str + expected_sqlstate: str + + +class VerificationError(RuntimeError): + """A deliberately sanitized, machine-readable verification failure.""" + + def __init__( + self, + code: str, + check: str, + *, + expected_sqlstate: str | None = None, + observed_sqlstate: str | None = None, + ) -> None: + self.code = code + self.check = check + self.expected_sqlstate = expected_sqlstate + self.observed_sqlstate = observed_sqlstate + super().__init__(f"{code}:{check}") + + def as_dict(self) -> dict[str, str]: + payload = {"check": self.check, "code": self.code} + if self.expected_sqlstate is not None: + payload["expected_sqlstate"] = self.expected_sqlstate + if self.observed_sqlstate is not None: + payload["observed_sqlstate"] = self.observed_sqlstate + return payload + + +def subprocess_runner( + command: list[str], + env: Mapping[str, str], + timeout: int, + discard_stdout: bool, +) -> CommandResult: + completed = subprocess.run( + command, + check=False, + text=True, + env=dict(env), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL if discard_stdout else subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + return CommandResult( + returncode=completed.returncode, + stdout="" if discard_stdout else (completed.stdout or ""), + stderr=completed.stderr or "", + ) + + +def validate_run_id(value: str) -> str: + if not RUN_ID_RE.fullmatch(value): + raise ValueError("run id must be 8-64 lowercase letters, digits, or hyphens") + return value + + +def _run_id_arg(value: str) -> str: + try: + return validate_run_id(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(str(exc)) from None + + +def _base_child_environment(_source: Mapping[str, str]) -> dict[str, str]: + return { + "HOME": "/home/teleo", + "LANG": "C", + "LC_ALL": "C", + "PATH": CHILD_PATH, + } + + +def _gcloud_environment(source: Mapping[str, str], config_dir: Path) -> dict[str, str]: + env = _base_child_environment(source) + env.update( + { + "CLOUDSDK_CONFIG": str(config_dir), + "CLOUDSDK_CORE_DISABLE_PROMPTS": "1", + "CLOUDSDK_CORE_PROJECT": PROJECT_ID, + } + ) + return env + + +def _psql_environment(source: Mapping[str, str], password: str) -> dict[str, str]: + env = _base_child_environment(source) + env["PGPASSWORD"] = password + return env + + +def _call( + command: list[str], + *, + env: Mapping[str, str], + runner: Runner, + check: str, + discard_stdout: bool = False, +) -> CommandResult: + try: + return runner(command, env, COMMAND_TIMEOUT_SECONDS, discard_stdout) + except FileNotFoundError: + raise VerificationError("command_not_found", check) from None + except subprocess.TimeoutExpired: + raise VerificationError("command_timed_out", check) from None + except OSError: + raise VerificationError("command_os_error", check) from None + + +def _secret_command(secret_name: str) -> list[str]: + return [ + GCLOUD_BIN, + "secrets", + "versions", + "access", + "latest", + f"--secret={secret_name}", + f"--project={PROJECT_ID}", + "--quiet", + ] + + +def _read_scoped_password(*, env: Mapping[str, str], runner: Runner) -> str: + result = _call( + _secret_command(SCOPED_PASSWORD_SECRET), + env=env, + runner=runner, + check="scoped_secret_access", + ) + if result.returncode != 0: + raise VerificationError("scoped_secret_access_failed", "scoped_secret_access") + + password = result.stdout[:-1] if result.stdout.endswith("\n") else result.stdout + if password.endswith("\r"): + password = password[:-1] + if not password or len(password) > 4096 or any(character in password for character in ("\x00", "\r", "\n")): + raise VerificationError("scoped_secret_value_invalid", "scoped_secret_access") + return password + + +def _assert_administrator_secret_denied(*, env: Mapping[str, str], runner: Runner) -> dict[str, Any]: + result = _call( + _secret_command(ADMINISTRATOR_PASSWORD_SECRET), + env=env, + runner=runner, + check="administrator_secret_access", + discard_stdout=True, + ) + if result.returncode == 0: + raise VerificationError("administrator_secret_unexpectedly_readable", "administrator_secret_access") + + permission_denied = re.search(r"\bPERMISSION_DENIED\b", result.stderr, re.IGNORECASE) is not None + exact_permission = IAM_PERMISSION in result.stderr.lower() + if not permission_denied or not exact_permission: + raise VerificationError("administrator_secret_denial_not_iam_permission", "administrator_secret_access") + return { + "classification": "iam_permission_denied", + "permission": IAM_PERMISSION, + "result": "denied", + "stdout_discarded": True, + } + + +def _conninfo(database: str) -> str: + return ( + f"host={PRIVATE_CLOUDSQL_HOST} port={PRIVATE_CLOUDSQL_PORT} dbname={database} " + f"user={RUNTIME_DATABASE_ROLE} sslmode=require connect_timeout=10 " + "application_name=leoclean_runtime_permission_verifier" + ) + + +def _psql_command(database: str, sql: str) -> list[str]: + return [ + PSQL_BIN, + "--no-psqlrc", + "--no-password", + "--quiet", + "--tuples-only", + "--no-align", + "--set=ON_ERROR_STOP=1", + "--set=VERBOSITY=verbose", + f"--dbname={_conninfo(database)}", + f"--command={sql}", + ] + + +def _single_output_line(result: CommandResult, check: str) -> str: + lines = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if len(lines) != 1: + raise VerificationError("unexpected_psql_output_shape", check) + return lines[0] + + +def _run_psql_success( + *, + check: str, + database: str, + sql: str, + env: Mapping[str, str], + runner: Runner, +) -> str: + result = _call( + _psql_command(database, sql), + env=env, + runner=runner, + check=check, + ) + if result.returncode != 0: + observed = sorted(set(SQLSTATE_RE.findall(result.stderr))) + raise VerificationError( + "psql_check_failed", + check, + observed_sqlstate=",".join(observed) if observed else "missing", + ) + return _single_output_line(result, check) + + +def _parse_json_object(raw: str, check: str) -> dict[str, Any]: + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + raise VerificationError("psql_json_invalid", check) from None + if not isinstance(parsed, dict): + raise VerificationError("psql_json_not_object", check) + return parsed + + +def _parse_zero_count(raw: str, check: str) -> int: + if raw != "0": + raise VerificationError("canary_row_count_nonzero", check) + return 0 + + +def _sql_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def normalize_stage_function_source(source: str) -> str: + normalized = source.replace("\r\n", "\n").replace("\r", "\n") + return normalized.strip("\n") + "\n" + + +def _identity_sql() -> str: + return """ +select pg_catalog.jsonb_build_object( + 'database', pg_catalog.current_database(), + 'current_user', current_user, + 'session_user', session_user, + 'server_addr', pg_catalog.inet_server_addr()::text, + 'server_port', pg_catalog.inet_server_port(), + 'ssl', coalesce( + (select ssl from pg_catalog.pg_stat_ssl where pid = pg_catalog.pg_backend_pid()), + false + ), + 'ssl_version', coalesce( + (select version from pg_catalog.pg_stat_ssl where pid = pg_catalog.pg_backend_pid()), + '' + ) +)::text; +""".strip() + + +def _database_role_posture_sql(role_name: str) -> str: + return f""" +select pg_catalog.jsonb_build_object( + 'role', role_row.rolname, + 'can_login', role_row.rolcanlogin, + 'is_superuser', role_row.rolsuper, + 'can_create_db', role_row.rolcreatedb, + 'can_create_role', role_row.rolcreaterole, + 'inherits_privileges', role_row.rolinherit, + 'can_replicate', role_row.rolreplication, + 'bypasses_rls', role_row.rolbypassrls, + 'connection_limit', role_row.rolconnlimit, + 'direct_membership_edges', ( + select pg_catalog.count(*) + from pg_catalog.pg_auth_members membership + where membership.roleid = role_row.oid + or membership.member = role_row.oid + ) +)::text +from pg_catalog.pg_roles role_row +where role_row.rolname = {_sql_literal(role_name)}; +""".strip() + + +def _role_posture_sql() -> str: + return _database_role_posture_sql(RUNTIME_DATABASE_ROLE) + + +def _stage_owner_role_posture_sql() -> str: + return _database_role_posture_sql(STAGE_OWNER_DATABASE_ROLE) + + +def _function_privilege_posture_sql() -> str: + values_sql = ",\n ".join( + f"({_sql_literal(name)}, {_sql_literal(signature)})" + for name, signature, _expected_exists, _expected_execute in FUNCTION_PRIVILEGE_EXPECTATIONS + ) + return f""" +with expected(check_name, signature) as ( + values + {values_sql} +), resolved as ( + select check_name, + signature, + pg_catalog.to_regprocedure(signature) as function_oid + from expected +) +select pg_catalog.jsonb_object_agg( + check_name, + pg_catalog.jsonb_build_object( + 'exists', function_oid is not null, + 'execute', coalesce( + pg_catalog.has_function_privilege(current_user, function_oid::pg_catalog.oid, 'EXECUTE'), + false + ) + ) +)::text +from resolved; +""".strip() + + +def _stage_function_definition_sql() -> str: + return f""" +with target as ( + select pg_catalog.to_regprocedure({_sql_literal(STAGE_FUNCTION_SIGNATURE)})::pg_catalog.oid as oid +), function_row as ( + select target.oid, + function_catalog.proowner, + function_catalog.prolang, + function_catalog.prokind, + function_catalog.prosecdef, + function_catalog.proleakproof, + function_catalog.proisstrict, + function_catalog.proretset, + function_catalog.provolatile, + function_catalog.proparallel, + function_catalog.pronargdefaults, + function_catalog.proargnames, + function_catalog.proargmodes, + function_catalog.proargtypes, + function_catalog.proconfig, + function_catalog.proacl, + function_catalog.prosrc, + owner_role.rolname as owner_name, + language_row.lanname as language_name + from target + left join pg_catalog.pg_proc function_catalog on function_catalog.oid = target.oid + left join pg_catalog.pg_roles owner_role on owner_role.oid = function_catalog.proowner + left join pg_catalog.pg_language language_row on language_row.oid = function_catalog.prolang +), runtime_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(RUNTIME_DATABASE_ROLE)} +), owner_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(STAGE_OWNER_DATABASE_ROLE)} +) +select pg_catalog.jsonb_build_object( + 'exists', function_row.oid is not null, + 'owner', function_row.owner_name, + 'language', function_row.language_name, + 'kind', function_row.prokind, + 'security_definer', function_row.prosecdef, + 'leakproof', function_row.proleakproof, + 'strict', function_row.proisstrict, + 'returns_set', function_row.proretset, + 'volatility', function_row.provolatile, + 'parallel', function_row.proparallel, + 'argument_defaults', function_row.pronargdefaults, + 'argument_names', function_row.proargnames, + 'argument_modes', function_row.proargmodes, + 'argument_types', ( + select pg_catalog.jsonb_agg(pg_catalog.format_type(argument_type, null) order by ordinal) + from pg_catalog.unnest(function_row.proargtypes::pg_catalog.oid[]) + with ordinality argument(argument_type, ordinal) + ), + 'result', pg_catalog.pg_get_function_result(function_row.oid), + 'configuration', function_row.proconfig, + 'runtime_execute', coalesce( + pg_catalog.has_function_privilege(current_user, function_row.oid, 'EXECUTE'), + false + ), + 'acl_exact', coalesce(( + select pg_catalog.count(*) filter ( + where acl.grantee = runtime_role.oid + and acl.privilege_type = 'EXECUTE' + and not acl.is_grantable + ) = 1 + and pg_catalog.count(*) filter ( + where not ( + acl.grantee in (runtime_role.oid, owner_role.oid) + and acl.privilege_type = 'EXECUTE' + and (acl.grantee = owner_role.oid or not acl.is_grantable) + ) + ) = 0 + from runtime_role + cross join owner_role + cross join lateral pg_catalog.aclexplode( + coalesce(function_row.proacl, pg_catalog.acldefault('f', function_row.proowner)) + ) acl + ), false), + 'source', function_row.prosrc +)::text +from function_row; +""".strip() + + +def _catalog_privilege_posture_sql() -> str: + allowlist_values = ",\n ".join( + f"({_sql_literal(schema)}, {_sql_literal(relation)})" for schema, relation in READ_TABLE_ALLOWLIST + ) + owner_insert_values = ",\n ".join( + f"({_sql_literal(column_name)})" for column_name in STAGE_OWNER_INSERT_COLUMNS + ) + return f""" +with runtime_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(RUNTIME_DATABASE_ROLE)} +), owner_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(STAGE_OWNER_DATABASE_ROLE)} +), stage_function as ( + select pg_catalog.to_regprocedure({_sql_literal(STAGE_FUNCTION_SIGNATURE)})::pg_catalog.oid as oid +), app_schemas as ( + select oid, nspname + from pg_catalog.pg_namespace + where nspname <> 'information_schema' + and nspname !~ '^pg_' +), allowed_select(nspname, relname) as ( + values + {allowlist_values} +), allowed_relation as ( + select allowed_select.nspname, + allowed_select.relname, + relation.oid + from allowed_select + left join pg_catalog.pg_namespace namespace + on namespace.nspname = allowed_select.nspname + left join pg_catalog.pg_class relation + on relation.relnamespace = namespace.oid + and relation.relname = allowed_select.relname + and relation.relkind in ('r', 'p', 'v', 'm', 'f') +), owner_insert_column(attname) as ( + values + {owner_insert_values} +) +select pg_catalog.jsonb_build_object( + 'canonical_connect', coalesce( + pg_catalog.has_database_privilege(current_user, pg_catalog.current_database(), 'CONNECT'), + false + ), + 'runtime_connect_acl_exact', coalesce(( + select pg_catalog.count(*) filter ( + where database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + and acl.privilege_type = 'CONNECT' + and not acl.is_grantable + ) = 1 + from pg_catalog.pg_database database_row + cross join runtime_role + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) + and acl.grantee = runtime_role.oid + ), false), + 'unexpected_direct_database_acl_entries', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + cross join runtime_role + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) + and acl.grantee = runtime_role.oid + and not ( + database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + and acl.privilege_type = 'CONNECT' + and not acl.is_grantable + ) + ), + 'owner_direct_database_acl_entries', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + cross join owner_role + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) + and acl.grantee = owner_role.oid + ), + 'database_create_grants', ( + select pg_catalog.count(*)::int + from (values ({_sql_literal(CANONICAL_DATABASE)}), ({_sql_literal(LEGACY_DATABASE)})) database_name(name) + where pg_catalog.has_database_privilege(current_user, database_name.name, 'CREATE') + ), + 'owner_database_create_grants', ( + select pg_catalog.count(*)::int + from (values ({_sql_literal(CANONICAL_DATABASE)}), ({_sql_literal(LEGACY_DATABASE)})) database_name(name) + where pg_catalog.has_database_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + database_name.name, + 'CREATE' + ) + ), + 'owned_database_objects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_shdepend dependency + join runtime_role on runtime_role.oid = dependency.refobjid + where dependency.refclassid = 'pg_catalog.pg_authid'::pg_catalog.regclass + and dependency.deptype = 'o' + ), + 'owner_unexpected_owned_database_objects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_shdepend dependency + join owner_role on owner_role.oid = dependency.refobjid + cross join stage_function + where dependency.refclassid = 'pg_catalog.pg_authid'::pg_catalog.regclass + and dependency.deptype = 'o' + and not ( + dependency.dbid = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = pg_catalog.current_database() + ) + and dependency.classid = 'pg_catalog.pg_proc'::pg_catalog.regclass + and dependency.objid = stage_function.oid + and dependency.objsubid = 0 + ) + ), + 'runtime_schema_usage', coalesce( + pg_catalog.has_schema_privilege(current_user, 'public', 'USAGE') + and pg_catalog.has_schema_privilege(current_user, 'kb_stage', 'USAGE'), + false + ), + 'owner_schema_usage', coalesce( + pg_catalog.has_schema_privilege({_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, 'public', 'USAGE') + and pg_catalog.has_schema_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + 'kb_stage', + 'USAGE' + ), + false + ), + 'unexpected_schema_usage', ( + select pg_catalog.count(*)::int + from app_schemas + where app_schemas.nspname not in ('public', 'kb_stage') + and pg_catalog.has_schema_privilege(current_user, app_schemas.oid, 'USAGE') + ), + 'owner_unexpected_schema_usage', ( + select pg_catalog.count(*)::int + from app_schemas + where app_schemas.nspname not in ('public', 'kb_stage') + and pg_catalog.has_schema_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + app_schemas.oid, + 'USAGE' + ) + ), + 'schema_create_grants', ( + select pg_catalog.count(*)::int + from app_schemas + where pg_catalog.has_schema_privilege(current_user, app_schemas.oid, 'CREATE') + ), + 'owner_schema_create_grants', ( + select pg_catalog.count(*)::int + from app_schemas + where pg_catalog.has_schema_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + app_schemas.oid, + 'CREATE' + ) + ), + 'missing_allowed_table_selects', ( + select pg_catalog.count(*)::int + from allowed_relation + where allowed_relation.oid is null + or not coalesce( + pg_catalog.has_table_privilege(current_user, allowed_relation.oid, 'SELECT'), + false + ) + ), + 'table_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege(current_user, relation.oid, privilege.name) + ), + 'column_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege(current_user, relation.oid, attribute.attnum, privilege.name) + ), + 'unexpected_table_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + left join allowed_select + on allowed_select.nspname = namespace.nspname + and allowed_select.relname = relation.relname + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege(current_user, relation.oid, 'SELECT') + and allowed_select.relname is null + ), + 'unexpected_column_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + left join allowed_select + on allowed_select.nspname = namespace.nspname + and allowed_select.relname = relation.relname + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege(current_user, relation.oid, attribute.attnum, 'SELECT') + and allowed_select.relname is null + ), + 'sequence_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where relation.relkind = 'S' + and pg_catalog.has_sequence_privilege(current_user, relation.oid, privilege.name) + ), + 'unexpected_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join stage_function + where function_row.oid is distinct from stage_function.oid + and pg_catalog.has_function_privilege(current_user, function_row.oid, 'EXECUTE') + ), + 'unexpected_security_definer_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join stage_function + where function_row.prosecdef + and function_row.oid is distinct from stage_function.oid + and pg_catalog.has_function_privilege(current_user, function_row.oid, 'EXECUTE') + ), + 'owner_grant_contract', coalesce( + not pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + pg_catalog.to_regclass('public.agents'), + 'SELECT' + ) + and pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + pg_catalog.to_regclass('kb_stage.kb_proposals'), + 'SELECT' + ) + and not pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + pg_catalog.to_regclass('kb_stage.kb_proposals'), + 'INSERT' + ), + false + ), + 'owner_missing_agent_select_columns', ( + select pg_catalog.count(*)::int + from (values ('id'), ('handle')) required_column(attname) + left join pg_catalog.pg_attribute attribute + on attribute.attrelid = pg_catalog.to_regclass('public.agents') + and attribute.attname = required_column.attname + and attribute.attnum > 0 + and not attribute.attisdropped + where attribute.attnum is null + or not coalesce( + pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + attribute.attrelid, + attribute.attnum, + 'SELECT' + ), + false + ) + ), + 'owner_missing_proposal_insert_columns', ( + select pg_catalog.count(*)::int + from owner_insert_column + left join pg_catalog.pg_attribute attribute + on attribute.attrelid = pg_catalog.to_regclass('kb_stage.kb_proposals') + and attribute.attname = owner_insert_column.attname + and attribute.attnum > 0 + and not attribute.attisdropped + where attribute.attnum is null + or not coalesce( + pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + attribute.attrelid, + attribute.attnum, + 'INSERT' + ), + false + ) + ), + 'owner_table_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + privilege.name + ) + ), + 'owner_unexpected_column_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + attribute.attnum, + privilege.name + ) + and not ( + privilege.name = 'INSERT' + and namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' + and attribute.attname in (select attname from owner_insert_column) + ) + ), + 'owner_unexpected_table_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + 'SELECT' + ) + and not (namespace.nspname = 'kb_stage' and relation.relname = 'kb_proposals') + ), + 'owner_unexpected_column_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + attribute.attnum, + 'SELECT' + ) + and not ( + (namespace.nspname = 'kb_stage' and relation.relname = 'kb_proposals') + or ( + namespace.nspname = 'public' + and relation.relname = 'agents' + and attribute.attname in ('id', 'handle') + ) + ) + ), + 'owner_sequence_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where relation.relkind = 'S' + and pg_catalog.has_sequence_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + privilege.name + ) + ), + 'owner_unexpected_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join stage_function + where function_row.oid is distinct from stage_function.oid + and pg_catalog.has_function_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + function_row.oid, + 'EXECUTE' + ) + ) +)::text; +""".strip() + + +def _allowed_read_sql() -> str: + return """ +select pg_catalog.jsonb_build_object( + 'claims_rows_sampled', (select pg_catalog.count(*) from (select 1 from public.claims limit 1) sampled), + 'proposal_rows_sampled', ( + select pg_catalog.count(*) from (select 1 from kb_stage.kb_proposals limit 1) sampled + ) +)::text; +""".strip() + + +def _legacy_catalog_privilege_posture_sql() -> str: + return f""" +with scoped_role(role_name, field_prefix) as ( + values + ({_sql_literal(RUNTIME_DATABASE_ROLE)}, 'runtime'), + ({_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, 'owner') +), restore_schema as ( + select oid + from pg_catalog.pg_namespace + where nspname = 'teleo_restore' +), posture as ( + select scoped_role.field_prefix, + ( + select pg_catalog.count(*)::int + from restore_schema + cross join (values ('USAGE'), ('CREATE')) privilege(name) + where pg_catalog.has_schema_privilege( + scoped_role.role_name, + restore_schema.oid, + privilege.name + ) + ) as schema_privileges, + ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join restore_schema on restore_schema.oid = relation.relnamespace + cross join ( + values ('SELECT'), ('INSERT'), ('UPDATE'), ('DELETE'), + ('TRUNCATE'), ('REFERENCES'), ('TRIGGER') + ) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege( + scoped_role.role_name, + relation.oid, + privilege.name + ) + ) as table_privileges, + ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join restore_schema on restore_schema.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('SELECT'), ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege( + scoped_role.role_name, + relation.oid, + attribute.attnum, + privilege.name + ) + ) as column_privileges, + ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join restore_schema on restore_schema.oid = relation.relnamespace + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where relation.relkind = 'S' + and pg_catalog.has_sequence_privilege( + scoped_role.role_name, + relation.oid, + privilege.name + ) + ) as sequence_privileges, + ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join restore_schema on restore_schema.oid = function_row.pronamespace + where pg_catalog.has_function_privilege( + scoped_role.role_name, + function_row.oid, + 'EXECUTE' + ) + ) as routine_privileges + from scoped_role +) +select pg_catalog.jsonb_build_object( + 'runtime_schema_privileges', max(schema_privileges) filter (where field_prefix = 'runtime'), + 'runtime_table_privileges', max(table_privileges) filter (where field_prefix = 'runtime'), + 'runtime_column_privileges', max(column_privileges) filter (where field_prefix = 'runtime'), + 'runtime_sequence_privileges', max(sequence_privileges) filter (where field_prefix = 'runtime'), + 'runtime_routine_privileges', max(routine_privileges) filter (where field_prefix = 'runtime'), + 'owner_schema_privileges', max(schema_privileges) filter (where field_prefix = 'owner'), + 'owner_table_privileges', max(table_privileges) filter (where field_prefix = 'owner'), + 'owner_column_privileges', max(column_privileges) filter (where field_prefix = 'owner'), + 'owner_sequence_privileges', max(sequence_privileges) filter (where field_prefix = 'owner'), + 'owner_routine_privileges', max(routine_privileges) filter (where field_prefix = 'owner') +)::text +from posture; +""".strip() + + +def _canary_count_sql(source_ref: str) -> str: + return f"select pg_catalog.count(*)::text from kb_stage.kb_proposals where source_ref = {_sql_literal(source_ref)};" + + +def _stage_sql(run_id: str, source_ref: str) -> str: + return f""" +begin; +with staged as ( + select kb_stage.stage_leoclean_proposal( + 'revise_claim', + 'runtime-permission-verification', + {_sql_literal(source_ref)}, + 'least-privilege transaction rollback verification', + pg_catalog.jsonb_build_object('run_id', {_sql_literal(run_id)}, 'verification', true) + ) as proposal +) +select pg_catalog.jsonb_build_object( + 'agent_id_matches', coalesce( + proposal ->> 'proposed_by_agent_id' = ( + select id::text from public.agents where handle = 'leo' + ), + false + ), + 'proposal', proposal +)::text +from staged; +rollback; +""".strip() + + +def _transactional(sql: str) -> str: + return f"begin;\n{sql.rstrip(';')};\nrollback;" + + +def _negative_checks(run_id: str, source_ref: str) -> tuple[NegativeCheck, ...]: + nil_uuid = "00000000-0000-0000-0000-000000000000" + return ( + NegativeCheck( + "proposal_insert", + CANONICAL_DATABASE, + _transactional("insert into kb_stage.kb_proposals select * from kb_stage.kb_proposals where false"), + "42501", + ), + NegativeCheck( + "proposal_update", + CANONICAL_DATABASE, + _transactional("update kb_stage.kb_proposals set status = status where false"), + "42501", + ), + NegativeCheck( + "proposal_delete", + CANONICAL_DATABASE, + _transactional("delete from kb_stage.kb_proposals where false"), + "42501", + ), + NegativeCheck( + "canonical_insert", + CANONICAL_DATABASE, + _transactional("insert into public.claims select * from public.claims where false"), + "42501", + ), + NegativeCheck( + "canonical_update", + CANONICAL_DATABASE, + _transactional("update public.claims set id = id where false"), + "42501", + ), + NegativeCheck( + "canonical_delete", + CANONICAL_DATABASE, + _transactional("delete from public.claims where false"), + "42501", + ), + NegativeCheck( + "forged_proposer_overload", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.stage_leoclean_proposal(" + f"'revise_claim', 'forged-principal', 'verification', {_sql_literal(source_ref)}, " + f"'must be unavailable', pg_catalog.jsonb_build_object('run_id', {_sql_literal(run_id)}))" + ), + "42883", + ), + NegativeCheck( + "reviewer_approve_function", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.approve_strict_proposal(" + f"'{nil_uuid}'::uuid, 'revise_claim', '{{}}'::jsonb, 'leo', 'must be denied')" + ), + "42501", + ), + NegativeCheck( + "apply_assert_function", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.assert_approved_proposal(" + f"'{nil_uuid}'::uuid, 'revise_claim', '{{}}'::jsonb, 'hash', " + f"'{nil_uuid}'::uuid, '2000-01-01T00:00:00Z'::timestamptz, 'leo')" + ), + "42501", + ), + NegativeCheck( + "apply_finish_function", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.finish_approved_proposal(" + f"'{nil_uuid}'::uuid, 'revise_claim', '{{}}'::jsonb, 'hash', " + f"'{nil_uuid}'::uuid, '2000-01-01T00:00:00Z'::timestamptz, 'leo', 'leo')" + ), + "42501", + ), + NegativeCheck( + "set_role_stage_owner", + CANONICAL_DATABASE, + _transactional("set role leoclean_kb_stage_owner"), + "42501", + ), + NegativeCheck( + "set_role_reviewer", + CANONICAL_DATABASE, + _transactional("set role kb_review"), + "42501", + ), + NegativeCheck( + "set_role_apply", + CANONICAL_DATABASE, + _transactional("set role kb_apply"), + "42501", + ), + NegativeCheck( + "set_role_administrator", + CANONICAL_DATABASE, + _transactional("set role postgres"), + "42501", + ), + NegativeCheck( + "legacy_teleo_restore_read", + LEGACY_DATABASE, + _transactional("select 1 from teleo_restore.response_audit limit 1"), + "42501", + ), + ) + + +def _expect_sqlstate( + check: NegativeCheck, + *, + env: Mapping[str, str], + runner: Runner, +) -> dict[str, str]: + result = _call( + _psql_command(check.database, check.sql), + env=env, + runner=runner, + check=check.name, + ) + if result.returncode == 0: + raise VerificationError( + "negative_permission_unexpectedly_succeeded", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate="success", + ) + + observed = SQLSTATE_RE.findall(result.stderr) + unique_observed = sorted(set(observed)) + if not observed: + raise VerificationError( + "negative_permission_sqlstate_missing", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate="missing", + ) + if unique_observed != [check.expected_sqlstate]: + raise VerificationError( + "negative_permission_sqlstate_mismatch", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate=",".join(unique_observed), + ) + return { + "check": check.name, + "expected_sqlstate": check.expected_sqlstate, + "observed_sqlstate": check.expected_sqlstate, + "result": "denied", + } + + +def _assert_identity(identity: dict[str, Any]) -> dict[str, Any]: + expected = { + "database": CANONICAL_DATABASE, + "current_user": RUNTIME_DATABASE_ROLE, + "session_user": RUNTIME_DATABASE_ROLE, + "server_addr": PRIVATE_CLOUDSQL_HOST, + "server_port": PRIVATE_CLOUDSQL_PORT, + "ssl": True, + } + if any(identity.get(key) != value for key, value in expected.items()): + raise VerificationError("private_ssl_identity_mismatch", "database_identity") + ssl_version = identity.get("ssl_version") + if not isinstance(ssl_version, str) or not ssl_version: + raise VerificationError("ssl_version_missing", "database_identity") + return {**expected, "ssl_version": ssl_version} + + +def _assert_role_posture(posture: dict[str, Any]) -> dict[str, Any]: + expected_booleans = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": True, + "can_replicate": False, + "inherits_privileges": False, + "is_superuser": False, + } + if posture.get("role") != RUNTIME_DATABASE_ROLE: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + for key, expected in expected_booleans.items(): + if posture.get(key) is not expected: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + + connection_limit = posture.get("connection_limit") + membership_edges = posture.get("direct_membership_edges") + if isinstance(connection_limit, bool) or not isinstance(connection_limit, int) or connection_limit != 8: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + if isinstance(membership_edges, bool) or not isinstance(membership_edges, int) or membership_edges != 0: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + + return { + "role": RUNTIME_DATABASE_ROLE, + **expected_booleans, + "connection_limit": connection_limit, + "direct_membership_edges": membership_edges, + } + + +def _assert_stage_owner_role_posture(posture: dict[str, Any]) -> dict[str, Any]: + expected_booleans = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": False, + "can_replicate": False, + "inherits_privileges": False, + "is_superuser": False, + } + if posture.get("role") != STAGE_OWNER_DATABASE_ROLE: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + for key, expected in expected_booleans.items(): + if posture.get(key) is not expected: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + + connection_limit = posture.get("connection_limit") + membership_edges = posture.get("direct_membership_edges") + if isinstance(connection_limit, bool) or not isinstance(connection_limit, int) or connection_limit != -1: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + if isinstance(membership_edges, bool) or not isinstance(membership_edges, int) or membership_edges != 0: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + + return { + "role": STAGE_OWNER_DATABASE_ROLE, + **expected_booleans, + "connection_limit": connection_limit, + "direct_membership_edges": membership_edges, + } + + +def _assert_function_privilege_posture(posture: dict[str, Any]) -> dict[str, dict[str, Any]]: + sanitized: dict[str, dict[str, Any]] = {} + for name, signature, expected_exists, expected_execute in FUNCTION_PRIVILEGE_EXPECTATIONS: + observed = posture.get(name) + if not isinstance(observed, dict): + raise VerificationError("function_privilege_posture_mismatch", "function_privilege_posture") + exists = observed.get("exists") + execute = observed.get("execute") + if exists is not expected_exists or execute is not expected_execute: + raise VerificationError("function_privilege_posture_mismatch", "function_privilege_posture") + sanitized[name] = { + "execute": expected_execute, + "exists": expected_exists, + "signature": signature, + } + return sanitized + + +def _assert_stage_function_definition(posture: dict[str, Any]) -> dict[str, Any]: + if posture.get("exists") is not True: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + for field, expected in STAGE_FUNCTION_METADATA_EXPECTATIONS: + observed = posture.get(field) + if type(observed) is not type(expected) or observed != expected: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + if posture.get("argument_modes") is not None: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + if posture.get("configuration") != ["search_path=pg_catalog, pg_temp"]: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + for field in ("acl_exact", "runtime_execute"): + if posture.get(field) is not True: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + + source = posture.get("source") + if not isinstance(source, str) or len(source.encode("utf-8")) > MAX_STAGE_FUNCTION_SOURCE_BYTES: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + source_sha256 = hashlib.sha256(normalize_stage_function_source(source).encode("utf-8")).hexdigest() + if source_sha256 != EXPECTED_STAGE_FUNCTION_SOURCE_SHA256: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + + return { + "acl_exact": True, + "argument_modes": None, + "configuration": ["search_path=pg_catalog, pg_temp"], + "exists": True, + **{field: expected for field, expected in STAGE_FUNCTION_METADATA_EXPECTATIONS}, + "runtime_execute": True, + "signature": STAGE_FUNCTION_SIGNATURE, + "source_sha256": source_sha256, + } + + +def _assert_catalog_privilege_posture(posture: dict[str, Any]) -> dict[str, int | bool]: + sanitized: dict[str, int | bool] = {} + for field in CATALOG_ZERO_COUNT_FIELDS: + value = posture.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value != 0: + raise VerificationError("catalog_privilege_posture_mismatch", "catalog_privilege_posture") + sanitized[field] = 0 + for field in CATALOG_TRUE_FIELDS: + if posture.get(field) is not True: + raise VerificationError("catalog_privilege_posture_mismatch", "catalog_privilege_posture") + sanitized[field] = True + return sanitized + + +def _assert_legacy_catalog_privilege_posture(posture: dict[str, Any]) -> dict[str, int]: + sanitized: dict[str, int] = {} + for field in LEGACY_CATALOG_ZERO_COUNT_FIELDS: + value = posture.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value != 0: + raise VerificationError("legacy_catalog_privilege_posture_mismatch", "legacy_catalog_privileges") + sanitized[field] = 0 + return sanitized + + +def _assert_allowed_reads(readback: dict[str, Any]) -> dict[str, int]: + sanitized: dict[str, int] = {} + for key in ("claims_rows_sampled", "proposal_rows_sampled"): + value = readback.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value not in (0, 1): + raise VerificationError("allowed_read_invalid", "canonical_allowed_reads") + sanitized[key] = value + return sanitized + + +def _assert_stage_result(staged: dict[str, Any], source_ref: str) -> dict[str, Any]: + proposal = staged.get("proposal") + if not isinstance(proposal, dict) or staged.get("agent_id_matches") is not True: + raise VerificationError("staged_proposal_agent_mismatch", "rolled_back_proposal_stage") + expected = { + "proposed_by_handle": "leo", + "source_ref": source_ref, + "status": "pending_review", + } + if any(proposal.get(key) != value for key, value in expected.items()): + raise VerificationError("staged_proposal_contract_mismatch", "rolled_back_proposal_stage") + proposed_by_agent_id = proposal.get("proposed_by_agent_id") + if not isinstance(proposed_by_agent_id, str) or not proposed_by_agent_id: + raise VerificationError("staged_proposal_agent_missing", "rolled_back_proposal_stage") + return {**expected, "agent_id_matches": True, "proposed_by_agent_id": proposed_by_agent_id} + + +def verify_runtime_permissions( + run_id: str, + *, + runner: Runner = subprocess_runner, + base_env: Mapping[str, str] | None = None, + effective_user: str | None = None, +) -> dict[str, Any]: + try: + run_id = validate_run_id(run_id) + except ValueError: + raise VerificationError("run_id_invalid", "arguments") from None + + if effective_user is None: + try: + effective_user = pwd.getpwuid(os.geteuid()).pw_name + except KeyError: + raise VerificationError("unix_user_unknown", "runtime_user") from None + if effective_user != RUNTIME_UNIX_USER: + raise VerificationError("unexpected_unix_user", "runtime_user") + + source = os.environ if base_env is None else base_env + source_ref = f"leo-runtime-permission:{run_id}" + generated_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + with tempfile.TemporaryDirectory(prefix="leoclean-runtime-permission-gcloud-") as config_name: + config_dir = Path(config_name) + config_dir.chmod(0o700) + gcloud_env = _gcloud_environment(source, config_dir) + password = _read_scoped_password(env=gcloud_env, runner=runner) + psql_env = _psql_environment(source, password) + + identity = _assert_identity( + _parse_json_object( + _run_psql_success( + check="database_identity", + database=CANONICAL_DATABASE, + sql=_identity_sql(), + env=psql_env, + runner=runner, + ), + "database_identity", + ) + ) + role_posture = _assert_role_posture( + _parse_json_object( + _run_psql_success( + check="runtime_role_posture", + database=CANONICAL_DATABASE, + sql=_role_posture_sql(), + env=psql_env, + runner=runner, + ), + "runtime_role_posture", + ) + ) + stage_owner_role_posture = _assert_stage_owner_role_posture( + _parse_json_object( + _run_psql_success( + check="stage_owner_role_posture", + database=CANONICAL_DATABASE, + sql=_stage_owner_role_posture_sql(), + env=psql_env, + runner=runner, + ), + "stage_owner_role_posture", + ) + ) + function_privileges = _assert_function_privilege_posture( + _parse_json_object( + _run_psql_success( + check="function_privilege_posture", + database=CANONICAL_DATABASE, + sql=_function_privilege_posture_sql(), + env=psql_env, + runner=runner, + ), + "function_privilege_posture", + ) + ) + stage_function_definition = _assert_stage_function_definition( + _parse_json_object( + _run_psql_success( + check="stage_function_definition", + database=CANONICAL_DATABASE, + sql=_stage_function_definition_sql(), + env=psql_env, + runner=runner, + ), + "stage_function_definition", + ) + ) + catalog_privileges = _assert_catalog_privilege_posture( + _parse_json_object( + _run_psql_success( + check="catalog_privilege_posture", + database=CANONICAL_DATABASE, + sql=_catalog_privilege_posture_sql(), + env=psql_env, + runner=runner, + ), + "catalog_privilege_posture", + ) + ) + legacy_catalog_privileges = _assert_legacy_catalog_privilege_posture( + _parse_json_object( + _run_psql_success( + check="legacy_catalog_privileges", + database=LEGACY_DATABASE, + sql=_legacy_catalog_privilege_posture_sql(), + env=psql_env, + runner=runner, + ), + "legacy_catalog_privileges", + ) + ) + allowed_reads = _assert_allowed_reads( + _parse_json_object( + _run_psql_success( + check="canonical_allowed_reads", + database=CANONICAL_DATABASE, + sql=_allowed_read_sql(), + env=psql_env, + runner=runner, + ), + "canonical_allowed_reads", + ) + ) + rows_before = _parse_zero_count( + _run_psql_success( + check="canary_rows_before", + database=CANONICAL_DATABASE, + sql=_canary_count_sql(source_ref), + env=psql_env, + runner=runner, + ), + "canary_rows_before", + ) + staged = _assert_stage_result( + _parse_json_object( + _run_psql_success( + check="rolled_back_proposal_stage", + database=CANONICAL_DATABASE, + sql=_stage_sql(run_id, source_ref), + env=psql_env, + runner=runner, + ), + "rolled_back_proposal_stage", + ), + source_ref, + ) + rows_after = _parse_zero_count( + _run_psql_success( + check="canary_rows_after", + database=CANONICAL_DATABASE, + sql=_canary_count_sql(source_ref), + env=psql_env, + runner=runner, + ), + "canary_rows_after", + ) + + negative_permissions = [ + _expect_sqlstate(check, env=psql_env, runner=runner) for check in _negative_checks(run_id, source_ref) + ] + administrator_secret = _assert_administrator_secret_denied(env=gcloud_env, runner=runner) + + return { + "artifact": ARTIFACT_NAME, + "checks": { + "allowed_reads": allowed_reads, + "catalog_privileges": catalog_privileges, + "canary_rows_after": rows_after, + "canary_rows_before": rows_before, + "function_privileges": function_privileges, + "legacy_catalog_privileges": legacy_catalog_privileges, + "negative_permissions": negative_permissions, + "role_posture": role_posture, + "stage_function_definition": stage_function_definition, + "stage_owner_role_posture": stage_owner_role_posture, + "rolled_back_proposal": { + **staged, + "transaction": "rolled_back", + }, + }, + "current_tier": REQUIRED_TIER, + "database_identity": identity, + "execution": { + "canonical_writes_committed": False, + "service_independent": True, + "unix_user": RUNTIME_UNIX_USER, + }, + "generated_at_utc": generated_at, + "mode": "live_private_gcp_staging", + "required_tier": REQUIRED_TIER, + "run_id": run_id, + "schema_version": 1, + "secret_access": { + "administrator": { + **administrator_secret, + "secret": ADMINISTRATOR_PASSWORD_SECRET, + }, + "scoped_runtime": { + "result": "readable", + "secret": SCOPED_PASSWORD_SECRET, + "value_retained": False, + }, + }, + "status": "pass", + "safety": { + "canonical_write_committed": False, + "production_promotion_attempted": False, + "proposal_write_committed": False, + "secret_value_retained": False, + "telegram_send_attempted": False, + }, + "target": { + "database": CANONICAL_DATABASE, + "host": PRIVATE_CLOUDSQL_HOST, + "port": PRIVATE_CLOUDSQL_PORT, + "project": PROJECT_ID, + "role": RUNTIME_DATABASE_ROLE, + "sslmode": "require", + }, + } + + +def failure_receipt(run_id: str, error: VerificationError) -> dict[str, Any]: + return { + "artifact": ARTIFACT_NAME, + "current_tier": LOWER_TIER, + "error": error.as_dict(), + "generated_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"), + "required_tier": REQUIRED_TIER, + "run_id": run_id, + "schema_version": 1, + "status": "fail", + } + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n" + + +def write_receipt(path: Path, payload: Mapping[str, Any]) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + os.fchmod(descriptor, stat.S_IRUSR | stat.S_IWUSR) + with os.fdopen(descriptor, "w", encoding="utf-8", closefd=False) as handle: + handle.write(canonical_json(payload)) + handle.flush() + os.fsync(handle.fileno()) + finally: + os.close(descriptor) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-id", required=True, type=_run_id_arg) + parser.add_argument("--output", type=Path, help="optional sanitized JSON receipt (written mode 0600)") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + receipt = verify_runtime_permissions(args.run_id) + returncode = 0 + except VerificationError as exc: + receipt = failure_receipt(args.run_id, exc) + returncode = 1 + except Exception: + receipt = failure_receipt(args.run_id, VerificationError("internal_error", "verifier")) + returncode = 1 + + if args.output is not None: + try: + write_receipt(args.output, receipt) + except OSError: + receipt = failure_receipt(args.run_id, VerificationError("output_write_failed", "output")) + returncode = 1 + sys.stdout.write(canonical_json(receipt)) + return returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/verify_gcp_leoclean_service_environment.py b/ops/verify_gcp_leoclean_service_environment.py new file mode 100755 index 0000000..8dff5a1 --- /dev/null +++ b/ops/verify_gcp_leoclean_service_environment.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Verify the effective Leo service environment without exposing its values.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +PROJECT_ID = "teleo-501523" +PRIVATE_CLOUDSQL_HOST = "10.61.0.3" +PRIVATE_CLOUDSQL_PORT = "5432" +CANONICAL_DATABASE = "teleo_canonical" +RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime" +SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" +ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password" +RUNTIME_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +FORBIDDEN_EXACT_FIELDS = frozenset( + { + "ALL_PROXY", + "BASH_ENV", + "BASHOPTS", + "CDPATH", + "CURL_CA_BUNDLE", + "ENV", + "GLOBIGNORE", + "GOOGLE_APPLICATION_CREDENTIALS", + "HTTPS_PROXY", + "HTTP_PROXY", + "IFS", + "NO_PROXY", + "PS4", + "REQUESTS_CA_BUNDLE", + "SHELLOPTS", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SSLKEYLOGFILE", + } +) + + +class VerificationError(RuntimeError): + """A sanitized failure safe to serialize in a public receipt.""" + + def __init__(self, code: str, fields: tuple[str, ...] = ()) -> None: + self.code = code + self.fields = tuple(sorted(set(fields))) + super().__init__(code) + + def as_dict(self) -> dict[str, object]: + payload: dict[str, object] = {"code": self.code} + if self.fields: + payload["fields"] = list(self.fields) + return payload + + +class _SanitizedArgumentParser(argparse.ArgumentParser): + def error(self, _message: str) -> None: + raise VerificationError("invalid_arguments") + + +def validate_pid(value: str | int) -> int: + """Return a positive process id without retaining the input on failure.""" + + if isinstance(value, bool) or not isinstance(value, (str, int)): + raise VerificationError("invalid_pid") + if isinstance(value, str) and (not value.isascii() or not value.isdecimal() or value.startswith("0")): + raise VerificationError("invalid_pid") + try: + pid = int(value) + except (TypeError, ValueError): + raise VerificationError("invalid_pid") from None + if pid <= 0: + raise VerificationError("invalid_pid") + return pid + + +def expected_environment() -> dict[str, str]: + """Build the exact reviewed environment contract.""" + + return { + "PATH": RUNTIME_PATH, + "TELEO_CANONICAL_CLOUDSQL_DB": CANONICAL_DATABASE, + "TELEO_CLOUDSQL_DB": CANONICAL_DATABASE, + "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK": "0", + "TELEO_CLOUDSQL_HOST": PRIVATE_CLOUDSQL_HOST, + "TELEO_CLOUDSQL_PASSWORD_SECRET": SCOPED_PASSWORD_SECRET, + "TELEO_CLOUDSQL_PORT": PRIVATE_CLOUDSQL_PORT, + "TELEO_CLOUDSQL_USER": RUNTIME_DATABASE_ROLE, + "TELEO_GCP_METADATA_ONLY": "1", + "TELEO_GCP_PROJECT": PROJECT_ID, + "TELEO_KB_MODE": "cloudsql", + } + + +def parse_environment(raw: bytes) -> dict[str, str]: + """Parse a Linux ``/proc/PID/environ`` block and reject ambiguity.""" + + if not isinstance(raw, bytes) or not raw or not raw.endswith(b"\0"): + raise VerificationError("environment_malformed") + + records = raw[:-1].split(b"\0") + if any(not record for record in records): + raise VerificationError("environment_malformed") + + environment: dict[str, str] = {} + try: + for record in records: + encoded_key, separator, encoded_value = record.partition(b"=") + if not separator or not encoded_key: + raise VerificationError("environment_malformed") + key = encoded_key.decode("utf-8", "strict") + value = encoded_value.decode("utf-8", "strict") + if key in environment: + raise VerificationError("environment_malformed") + environment[key] = value + except UnicodeDecodeError: + raise VerificationError("environment_malformed") from None + return environment + + +def _forbidden_field_labels(environment: Mapping[str, str]) -> tuple[str, ...]: + labels: set[str] = set() + for key in environment: + normalized = key.upper() + if normalized.startswith("PG"): + labels.add("PG*") + if normalized.startswith("CLOUDSDK_"): + labels.add("CLOUDSDK_*") + if normalized.startswith("LD_"): + labels.add("LD_*") + if normalized.startswith("BASH_FUNC_"): + labels.add("BASH_FUNC_*") + if normalized.startswith("PYTHON") and normalized != "PYTHONUNBUFFERED": + labels.add("PYTHON*") + if normalized in FORBIDDEN_EXACT_FIELDS: + labels.add(normalized) + return tuple(sorted(labels)) + + +def validate_environment(environment: Mapping[str, str]) -> tuple[str, ...]: + """Validate the effective environment and return only reviewed field names.""" + + if not isinstance(environment, Mapping) or any( + not isinstance(key, str) or not isinstance(value, str) for key, value in environment.items() + ): + raise VerificationError("environment_malformed") + + expected = expected_environment() + administrator_secret = ADMINISTRATOR_PASSWORD_SECRET.casefold() + if any(administrator_secret in value.casefold() for value in environment.values()): + raise VerificationError("administrator_secret_reference") + + forbidden_fields = _forbidden_field_labels(environment) + if forbidden_fields: + raise VerificationError("forbidden_environment_fields", forbidden_fields) + + mismatched = tuple( + sorted(key for key, expected_value in expected.items() if environment.get(key) != expected_value) + ) + if mismatched: + raise VerificationError("environment_mismatch", mismatched) + + return tuple(sorted(expected)) + + +def verify_process_environment( + pid: int, + *, + proc_root: Path = Path("/proc"), +) -> dict[str, object]: + """Read and verify one process environment, returning a sanitized receipt.""" + + validated_pid = validate_pid(pid) + raw = (proc_root / str(validated_pid) / "environ").read_bytes() + validated_fields = validate_environment(parse_environment(raw)) + return { + "runtime_environment": "scoped", + "status": "pass", + "validated_fields": list(validated_fields), + } + + +def failure_receipt(error: VerificationError) -> dict[str, object]: + return {"error": error.as_dict(), "status": "fail"} + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n" + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = _SanitizedArgumentParser(description=__doc__, add_help=False) + parser.add_argument("--pid", required=True) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + receipt = verify_process_environment(validate_pid(args.pid)) + returncode = 0 + except VerificationError as exc: + receipt = failure_receipt(exc) + returncode = 1 + except Exception: + receipt = failure_receipt(VerificationError("internal_error")) + returncode = 1 + + sys.stdout.write(canonical_json(receipt)) + return returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_gcp_generated_db_direct_claim_suite.py b/scripts/run_gcp_generated_db_direct_claim_suite.py index f2a3e31..f335731 100644 --- a/scripts/run_gcp_generated_db_direct_claim_suite.py +++ b/scripts/run_gcp_generated_db_direct_claim_suite.py @@ -47,7 +47,7 @@ DEFAULT_HOST = "10.61.0.3" DEFAULT_PROJECT = "teleo-501523" DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql" -REVIEWED_CLOUDSQL_TOOL_SHA256 = "2617bb501456758131ff41a6c77b3c462e21c28c4348da319b60defb051c7c87" +REVIEWED_CLOUDSQL_TOOL_SHA256 = "66fa84b3bd6fe67e102fbce4509ea28c3537ba5bb1091e1a1a457b7425366064" REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a" COUNT_READBACK_RE = re.compile( r"DB readback:\s*claims:\s*`?(?P\d+)`?;\s*" diff --git a/systemd/leoclean-gcp-prod-parallel-cloudsql.conf b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf index 1b267f5..68d4d34 100644 --- a/systemd/leoclean-gcp-prod-parallel-cloudsql.conf +++ b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf @@ -1,9 +1,19 @@ [Service] UnsetEnvironment=PGPASSWORD -RuntimeDirectory=leoclean-gcp-prod-parallel -RuntimeDirectoryMode=0700 -Environment=CLOUDSDK_CONFIG=/run/leoclean-gcp-prod-parallel +UnsetEnvironment=PGPASSFILE PGSERVICE PGHOST PGPORT PGDATABASE PGUSER PGOPTIONS +UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN +UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS CDPATH GLOBIGNORE IFS PS4 +UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT LD_DEBUG LD_DEBUG_OUTPUT LD_DYNAMIC_WEAK LD_HWCAP_MASK LD_ORIGIN_PATH LD_PROFILE LD_SHOW_AUXV LD_TRACE_LOADED_OBJECTS LD_USE_LOAD_BIAS LD_VERBOSE LD_WARN +UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT PYTHONUSERBASE PYTHONHTTPSVERIFY +UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy +UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE +ReadOnlyPaths=/home/teleo/.hermes +ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state /home/teleo/.hermes/profiles/leoclean/workspace +BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin +CapabilityBoundingSet=~CAP_SYS_ADMIN +Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin Environment=TELEO_KB_MODE=cloudsql +Environment=TELEO_GCP_METADATA_ONLY=1 Environment=TELEO_GCP_PROJECT=teleo-501523 Environment=TELEO_CLOUDSQL_HOST=10.61.0.3 Environment=TELEO_CLOUDSQL_PORT=5432 diff --git a/tests/test_gcp_leoclean_runtime_permissions.py b/tests/test_gcp_leoclean_runtime_permissions.py new file mode 100644 index 0000000..ca97995 --- /dev/null +++ b/tests/test_gcp_leoclean_runtime_permissions.py @@ -0,0 +1,797 @@ +import hashlib +import json +import os +import stat +import subprocess +from collections.abc import Mapping +from pathlib import Path + +import pytest + +from ops import verify_gcp_leoclean_runtime_permissions as verifier + +RUN_ID = "20260715-leo-security" +SOURCE_REF = f"leo-runtime-permission:{RUN_ID}" +SECRET_VALUE = "unit-only-secret-value%42" +AGENT_ID = "11111111-1111-4111-8111-111111111111" + + +class FakeRunner: + def __init__( + self, + *, + sqlstate_override: tuple[str, str] | None = None, + sqlstate_missing_for: str | None = None, + unexpected_success_for: str | None = None, + scoped_failure: bool = False, + administrator_mode: str = "denied", + identity_override: Mapping[str, object] | None = None, + role_posture_override: Mapping[str, object] | None = None, + stage_owner_role_posture_override: Mapping[str, object] | None = None, + function_posture_override: Mapping[str, Mapping[str, object]] | None = None, + stage_definition_override: Mapping[str, object] | None = None, + catalog_posture_override: Mapping[str, object] | None = None, + legacy_catalog_posture_override: Mapping[str, object] | None = None, + ) -> None: + self.sqlstate_override = sqlstate_override + self.sqlstate_missing_for = sqlstate_missing_for + self.unexpected_success_for = unexpected_success_for + self.scoped_failure = scoped_failure + self.administrator_mode = administrator_mode + self.identity_override = dict(identity_override or {}) + self.role_posture_override = dict(role_posture_override or {}) + self.stage_owner_role_posture_override = dict(stage_owner_role_posture_override or {}) + self.function_posture_override = { + name: dict(values) for name, values in (function_posture_override or {}).items() + } + self.stage_definition_override = dict(stage_definition_override or {}) + self.catalog_posture_override = dict(catalog_posture_override or {}) + self.legacy_catalog_posture_override = dict(legacy_catalog_posture_override or {}) + self.calls: list[tuple[list[str], dict[str, str], int, bool]] = [] + self.negative = {check.sql: check for check in verifier._negative_checks(RUN_ID, SOURCE_REF)} + + def __call__( + self, + command: list[str], + env: Mapping[str, str], + timeout: int, + discard_stdout: bool, + ) -> verifier.CommandResult: + self.calls.append((list(command), dict(env), timeout, discard_stdout)) + + if command[0] == verifier.GCLOUD_BIN: + scoped = f"--secret={verifier.SCOPED_PASSWORD_SECRET}" in command + if scoped: + if self.scoped_failure: + return verifier.CommandResult(1, SECRET_VALUE, f"scoped failure {SECRET_VALUE}") + return verifier.CommandResult(0, f"{SECRET_VALUE}\n", f"ignored warning {SECRET_VALUE}") + assert f"--secret={verifier.ADMINISTRATOR_PASSWORD_SECRET}" in command + assert discard_stdout is True + if self.administrator_mode == "success": + return verifier.CommandResult(0, SECRET_VALUE, f"ignored {SECRET_VALUE}") + if self.administrator_mode == "wrong-error": + return verifier.CommandResult(1, SECRET_VALUE, f"network unavailable {SECRET_VALUE}") + return verifier.CommandResult( + 1, + SECRET_VALUE, + ( + "ERROR: PERMISSION_DENIED: Permission " + f"'{verifier.IAM_PERMISSION}' denied for the administrator secret; {SECRET_VALUE}" + ), + ) + + assert command[0] == verifier.PSQL_BIN + assert discard_stdout is False + sql = next(part.removeprefix("--command=") for part in command if part.startswith("--command=")) + + if sql == verifier._identity_sql(): + identity: dict[str, object] = { + "current_user": verifier.RUNTIME_DATABASE_ROLE, + "database": verifier.CANONICAL_DATABASE, + "leak": SECRET_VALUE, + "server_addr": verifier.PRIVATE_CLOUDSQL_HOST, + "server_port": verifier.PRIVATE_CLOUDSQL_PORT, + "session_user": verifier.RUNTIME_DATABASE_ROLE, + "ssl": True, + "ssl_version": "TLSv1.3", + } + identity.update(self.identity_override) + return verifier.CommandResult(0, json.dumps(identity), f"ignored {SECRET_VALUE}") + if sql == verifier._allowed_read_sql(): + return verifier.CommandResult( + 0, + json.dumps( + { + "claims_rows_sampled": 1, + "leak": SECRET_VALUE, + "proposal_rows_sampled": 0, + } + ), + f"ignored {SECRET_VALUE}", + ) + if sql == verifier._role_posture_sql(): + role_posture: dict[str, object] = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": True, + "can_replicate": False, + "connection_limit": 8, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "leak": SECRET_VALUE, + "role": verifier.RUNTIME_DATABASE_ROLE, + } + role_posture.update(self.role_posture_override) + return verifier.CommandResult(0, json.dumps(role_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._stage_owner_role_posture_sql(): + owner_posture: dict[str, object] = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": False, + "can_replicate": False, + "connection_limit": -1, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "leak": SECRET_VALUE, + "role": verifier.STAGE_OWNER_DATABASE_ROLE, + } + owner_posture.update(self.stage_owner_role_posture_override) + return verifier.CommandResult(0, json.dumps(owner_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._function_privilege_posture_sql(): + function_posture: dict[str, dict[str, object]] = { + name: { + "execute": expected_execute, + "exists": expected_exists, + "leak": SECRET_VALUE, + } + for name, _signature, expected_exists, expected_execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS + } + for name, override in self.function_posture_override.items(): + function_posture.setdefault(name, {}).update(override) + function_posture["unexpected_leak"] = {"value": SECRET_VALUE} + return verifier.CommandResult(0, json.dumps(function_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._stage_function_definition_sql(): + stage_definition: dict[str, object] = { + "acl_exact": True, + "argument_modes": None, + "configuration": ["search_path=pg_catalog, pg_temp"], + "exists": True, + "leak": SECRET_VALUE, + "runtime_execute": True, + "source": verifier.EXPECTED_STAGE_FUNCTION_SOURCE, + **dict(verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS), + } + stage_definition.update(self.stage_definition_override) + return verifier.CommandResult(0, json.dumps(stage_definition), f"ignored {SECRET_VALUE}") + if sql == verifier._catalog_privilege_posture_sql(): + catalog_posture: dict[str, object] = { + field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS + } + catalog_posture.update( + { + "leak": SECRET_VALUE, + **{field: True for field in verifier.CATALOG_TRUE_FIELDS}, + } + ) + catalog_posture.update(self.catalog_posture_override) + return verifier.CommandResult(0, json.dumps(catalog_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._legacy_catalog_privilege_posture_sql(): + legacy_posture: dict[str, object] = { + field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS + } + legacy_posture["leak"] = SECRET_VALUE + legacy_posture.update(self.legacy_catalog_posture_override) + return verifier.CommandResult(0, json.dumps(legacy_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._canary_count_sql(SOURCE_REF): + return verifier.CommandResult(0, "0\n", f"ignored {SECRET_VALUE}") + if sql == verifier._stage_sql(RUN_ID, SOURCE_REF): + return verifier.CommandResult( + 0, + json.dumps( + { + "agent_id_matches": True, + "proposal": { + "leak": SECRET_VALUE, + "proposed_by_agent_id": AGENT_ID, + "proposed_by_handle": "leo", + "source_ref": SOURCE_REF, + "status": "pending_review", + }, + } + ), + f"ignored {SECRET_VALUE}", + ) + + check = self.negative[sql] + if check.name == self.unexpected_success_for: + return verifier.CommandResult(0, SECRET_VALUE, f"unexpected success {SECRET_VALUE}") + if check.name == self.sqlstate_missing_for: + return verifier.CommandResult(1, SECRET_VALUE, f"denied without state {SECRET_VALUE}") + observed = check.expected_sqlstate + if self.sqlstate_override is not None and check.name == self.sqlstate_override[0]: + observed = self.sqlstate_override[1] + return verifier.CommandResult( + 1, + SECRET_VALUE, + f"ERROR: {observed}: expected test denial; {SECRET_VALUE}\nLOCATION: test", + ) + + +def poisoned_environment() -> dict[str, str]: + return { + "CLOUDSDK_AUTH_ACCESS_TOKEN": "must-not-reach-gcloud", + "GOOGLE_APPLICATION_CREDENTIALS": "/tmp/must-not-reach-gcloud.json", + "HOME": "/home/teleo", + "LANG": "C.UTF-8", + "LD_PRELOAD": "/tmp/must-not-load.so", + "OTHER": "preserved", + "PGDATABASE": "wrong-db", + "PGHOST": "public.example.invalid", + "PGPASSWORD": "administrator-password", + "PGSERVICE": "broad-service", + "pgsslmode": "disable", + } + + +def run_success(runner: FakeRunner) -> dict[str, object]: + return verifier.verify_runtime_permissions( + RUN_ID, + runner=runner, + base_env=poisoned_environment(), + effective_user="teleo", + ) + + +def assert_child_boundaries(runner: FakeRunner) -> None: + assert runner.calls + for command, env, timeout, discard_stdout in runner.calls: + assert SECRET_VALUE not in "\n".join(command) + assert timeout == verifier.COMMAND_TIMEOUT_SECONDS + pg_keys = sorted(key for key in env if key.upper().startswith("PG")) + if command[0] == verifier.PSQL_BIN: + assert pg_keys == ["PGPASSWORD"] + assert env["PGPASSWORD"] == SECRET_VALUE + assert discard_stdout is False + else: + assert pg_keys == [] + assert "CLOUDSDK_AUTH_ACCESS_TOKEN" not in env + assert "GOOGLE_APPLICATION_CREDENTIALS" not in env + assert env["HOME"] == "/home/teleo" + assert env["LANG"] == "C" + assert env["LC_ALL"] == "C" + assert env["PATH"] == verifier.CHILD_PATH + assert "LD_PRELOAD" not in env + assert "OTHER" not in env + + +def test_live_receipt_contract_is_sanitized_and_uses_exact_child_environments() -> None: + runner = FakeRunner() + + receipt = run_success(runner) + serialized = verifier.canonical_json(receipt) + + assert receipt["status"] == "pass" + assert receipt["current_tier"] == verifier.REQUIRED_TIER + assert receipt["database_identity"] == { + "current_user": verifier.RUNTIME_DATABASE_ROLE, + "database": verifier.CANONICAL_DATABASE, + "server_addr": verifier.PRIVATE_CLOUDSQL_HOST, + "server_port": verifier.PRIVATE_CLOUDSQL_PORT, + "session_user": verifier.RUNTIME_DATABASE_ROLE, + "ssl": True, + "ssl_version": "TLSv1.3", + } + assert receipt["checks"]["rolled_back_proposal"] == { + "agent_id_matches": True, + "proposed_by_agent_id": AGENT_ID, + "proposed_by_handle": "leo", + "source_ref": SOURCE_REF, + "status": "pending_review", + "transaction": "rolled_back", + } + assert receipt["checks"]["canary_rows_before"] == 0 + assert receipt["checks"]["canary_rows_after"] == 0 + assert receipt["checks"]["role_posture"] == { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": True, + "can_replicate": False, + "connection_limit": 8, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "role": verifier.RUNTIME_DATABASE_ROLE, + } + assert receipt["checks"]["stage_owner_role_posture"] == { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": False, + "can_replicate": False, + "connection_limit": -1, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "role": verifier.STAGE_OWNER_DATABASE_ROLE, + } + assert receipt["checks"]["function_privileges"] == { + name: { + "execute": expected_execute, + "exists": expected_exists, + "signature": signature, + } + for name, signature, expected_exists, expected_execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS + } + assert receipt["checks"]["catalog_privileges"] == { + **{field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS}, + **{field: True for field in verifier.CATALOG_TRUE_FIELDS}, + } + assert receipt["checks"]["legacy_catalog_privileges"] == { + field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS + } + assert receipt["checks"]["stage_function_definition"] == { + "acl_exact": True, + "argument_modes": None, + "configuration": ["search_path=pg_catalog, pg_temp"], + "exists": True, + **dict(verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS), + "runtime_execute": True, + "signature": verifier.STAGE_FUNCTION_SIGNATURE, + "source_sha256": verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256, + } + assert len(receipt["checks"]["negative_permissions"]) == len(verifier._negative_checks(RUN_ID, SOURCE_REF)) + assert receipt["secret_access"]["administrator"]["classification"] == "iam_permission_denied" + assert receipt["secret_access"]["administrator"]["stdout_discarded"] is True + assert SECRET_VALUE not in serialized + assert "administrator-password" not in serialized + assert "stage_leoclean_proposal is restricted" not in serialized + assert_child_boundaries(runner) + + administrator_call = runner.calls[-1] + assert administrator_call[0][0] == verifier.GCLOUD_BIN + assert administrator_call[3] is True + + +@pytest.mark.parametrize( + ("attribute", "unsafe_value"), + [ + ("can_login", False), + ("is_superuser", True), + ("can_create_db", True), + ("can_create_role", True), + ("inherits_privileges", True), + ("can_replicate", True), + ("bypasses_rls", True), + ("connection_limit", 7), + ], +) +def test_any_unsafe_runtime_role_attribute_fails_closed(attribute: str, unsafe_value: object) -> None: + runner = FakeRunner(role_posture_override={attribute: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "runtime_role_posture_mismatch" + assert caught.value.check == "runtime_role_posture" + assert SECRET_VALUE not in str(caught.value) + + +def test_any_direct_runtime_role_membership_edge_fails_closed() -> None: + runner = FakeRunner(role_posture_override={"direct_membership_edges": 1}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "runtime_role_posture_mismatch" + assert caught.value.check == "runtime_role_posture" + + +@pytest.mark.parametrize( + ("attribute", "unsafe_value"), + [ + ("role", "unexpected-owner"), + ("can_login", True), + ("is_superuser", True), + ("can_create_db", True), + ("can_create_role", True), + ("inherits_privileges", True), + ("can_replicate", True), + ("bypasses_rls", True), + ("connection_limit", 0), + ("direct_membership_edges", 1), + ], +) +def test_any_unsafe_stage_owner_role_posture_fails_closed(attribute: str, unsafe_value: object) -> None: + runner = FakeRunner(stage_owner_role_posture_override={attribute: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_owner_role_posture_mismatch" + assert caught.value.check == "stage_owner_role_posture" + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize( + "function_name", + ["approve_strict_proposal", "assert_approved_proposal", "finish_approved_proposal"], +) +def test_reviewer_or_apply_execute_grant_fails_closed(function_name: str) -> None: + runner = FakeRunner(function_posture_override={function_name: {"execute": True}}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "function_privilege_posture_mismatch" + assert caught.value.check == "function_privilege_posture" + + +@pytest.mark.parametrize( + "function_name", + [ + "stage_leoclean_proposal_5_arg", + "approve_strict_proposal", + "assert_approved_proposal", + "finish_approved_proposal", + ], +) +def test_missing_expected_exact_function_fails_closed(function_name: str) -> None: + runner = FakeRunner(function_posture_override={function_name: {"execute": False, "exists": False}}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "function_privilege_posture_mismatch" + assert caught.value.check == "function_privilege_posture" + + +def test_forged_overload_presence_or_missing_stage_execute_fails_closed() -> None: + forged = FakeRunner(function_posture_override={"stage_leoclean_proposal_6_arg": {"exists": True}}) + missing_execute = FakeRunner(function_posture_override={"stage_leoclean_proposal_5_arg": {"execute": False}}) + + for runner in (forged, missing_execute): + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + assert caught.value.code == "function_privilege_posture_mismatch" + + +def _drifted_value(expected: object) -> object: + if isinstance(expected, bool): + return not expected + if isinstance(expected, int): + return expected + 1 + if isinstance(expected, str): + return f"{expected}-drift" + if isinstance(expected, list): + return [*expected, "drift"] + raise AssertionError(f"unsupported test expectation type: {type(expected).__name__}") + + +@pytest.mark.parametrize( + ("field", "unsafe_value"), + [ + (field, _drifted_value(expected)) + for field, expected in verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS + ], +) +def test_any_stage_function_metadata_drift_fails_closed(field: str, unsafe_value: object) -> None: + runner = FakeRunner(stage_definition_override={field: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_function_definition_mismatch" + assert caught.value.check == "stage_function_definition" + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize( + ("field", "unsafe_value"), + [ + ("exists", False), + ("argument_modes", ["i", "i", "i", "i", "i"]), + ("configuration", ["search_path=public"]), + ("acl_exact", False), + ("runtime_execute", False), + ("leakproof", 0), + ("argument_defaults", False), + ], +) +def test_any_stage_function_contract_drift_fails_closed(field: str, unsafe_value: object) -> None: + runner = FakeRunner(stage_definition_override={field: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_function_definition_mismatch" + assert caught.value.check == "stage_function_definition" + + +@pytest.mark.parametrize( + "unsafe_source", + [ + None, + 42, + verifier.EXPECTED_STAGE_FUNCTION_SOURCE.replace("return to_jsonb(staged);", "return '{}'::jsonb;"), + "x" * (verifier.MAX_STAGE_FUNCTION_SOURCE_BYTES + 1), + ], +) +def test_missing_malformed_oversized_or_semantically_drifted_stage_body_fails_closed( + unsafe_source: object, +) -> None: + runner = FakeRunner(stage_definition_override={"source": unsafe_source}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_function_definition_mismatch" + assert caught.value.check == "stage_function_definition" + failure = verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value)) + assert "stage_leoclean_proposal is restricted" not in failure + assert SECRET_VALUE not in failure + + +def test_stage_body_normalization_accepts_only_line_ending_and_framing_newline_changes() -> None: + crlf_source = "\r\n" + verifier.EXPECTED_STAGE_FUNCTION_SOURCE.strip("\n").replace("\n", "\r\n") + "\r\n" + + receipt = run_success(FakeRunner(stage_definition_override={"source": crlf_source})) + + assert ( + receipt["checks"]["stage_function_definition"]["source_sha256"] + == verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256 + ) + + +def test_embedded_stage_body_attestation_is_tied_to_the_provisioning_sql_source() -> None: + provisioning_sql = ( + Path(__file__).resolve().parents[1] / "ops" / "gcp_leoclean_runtime_role.sql" + ).read_text(encoding="utf-8") + function_anchor = "create or replace function kb_stage.stage_leoclean_proposal(" + assert provisioning_sql.count(function_anchor) == 1 + function_section = provisioning_sql.split(function_anchor, 1)[1] + assert function_section.count("as $function$") == 1 + provisioned_source = function_section.split("as $function$", 1)[1].split("$function$;", 1)[0] + + assert provisioned_source == verifier.EXPECTED_STAGE_FUNCTION_SOURCE + assert ( + hashlib.sha256(verifier.normalize_stage_function_source(provisioned_source).encode("utf-8")).hexdigest() + == verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256 + ) + + +def test_catalog_queries_use_exact_regprocedure_signatures_and_both_membership_directions() -> None: + function_sql = verifier._function_privilege_posture_sql() + role_sql = verifier._role_posture_sql() + owner_role_sql = verifier._stage_owner_role_posture_sql() + stage_definition_sql = verifier._stage_function_definition_sql() + catalog_sql = verifier._catalog_privilege_posture_sql() + legacy_sql = verifier._legacy_catalog_privilege_posture_sql() + + assert "pg_catalog.to_regprocedure(signature)" in function_sql + for _name, signature, _exists, _execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS: + assert signature in function_sql + assert "membership.roleid = role_row.oid" in role_sql + assert "membership.member = role_row.oid" in role_sql + assert "membership.roleid = role_row.oid" in owner_role_sql + assert "membership.member = role_row.oid" in owner_role_sql + assert verifier.STAGE_OWNER_DATABASE_ROLE in owner_role_sql + assert verifier.STAGE_FUNCTION_SIGNATURE in stage_definition_sql + assert "left join pg_catalog.pg_proc" in stage_definition_sql + assert "function_catalog.prosrc" in stage_definition_sql + assert "function_catalog.prokind" in stage_definition_sql + assert "function_catalog.proargtypes" in stage_definition_sql + assert "function_catalog.prosecdef" in stage_definition_sql + assert "pg_catalog.pg_get_function_result" in stage_definition_sql + assert "pg_catalog.aclexplode" in stage_definition_sql + assert "pg_catalog.pg_shdepend" in catalog_sql + assert "pg_catalog.has_table_privilege" in catalog_sql + assert "pg_catalog.has_column_privilege" in catalog_sql + assert "pg_catalog.has_sequence_privilege" in catalog_sql + assert "pg_catalog.has_function_privilege" in catalog_sql + assert "pg_catalog.aclexplode" in catalog_sql + assert "nspname !~ '^pg_'" in catalog_sql + assert "nspname <> 'information_schema'" in catalog_sql + assert verifier.STAGE_OWNER_DATABASE_ROLE in catalog_sql + assert "teleo_restore" in legacy_sql + assert "pg_catalog.has_schema_privilege" in legacy_sql + assert "pg_catalog.has_table_privilege" in legacy_sql + assert "pg_catalog.has_column_privilege" in legacy_sql + assert "pg_catalog.has_sequence_privilege" in legacy_sql + assert "pg_catalog.has_function_privilege" in legacy_sql + for schema, relation in verifier.READ_TABLE_ALLOWLIST: + assert schema in catalog_sql + assert relation in catalog_sql + + +@pytest.mark.parametrize("field", verifier.CATALOG_ZERO_COUNT_FIELDS) +def test_any_unexpected_catalog_privilege_fails_closed(field: str) -> None: + runner = FakeRunner(catalog_posture_override={field: 1}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "catalog_privilege_posture_mismatch" + assert caught.value.check == "catalog_privilege_posture" + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize("field", verifier.CATALOG_TRUE_FIELDS) +def test_any_missing_required_catalog_privilege_fails_closed(field: str) -> None: + runner = FakeRunner(catalog_posture_override={field: False}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "catalog_privilege_posture_mismatch" + + +@pytest.mark.parametrize("field", verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS) +def test_any_teleo_restore_catalog_privilege_fails_closed(field: str) -> None: + runner = FakeRunner(legacy_catalog_posture_override={field: 1}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "legacy_catalog_privilege_posture_mismatch" + assert caught.value.check == "legacy_catalog_privileges" + assert SECRET_VALUE not in str(caught.value) + + +def test_teleo_restore_catalog_and_behavioral_denial_are_both_exercised_on_legacy_database() -> None: + runner = FakeRunner() + + receipt = run_success(runner) + + legacy_catalog_call = next( + command + for command, _env, _timeout, _discard in runner.calls + if any(part == f"--command={verifier._legacy_catalog_privilege_posture_sql()}" for part in command) + ) + assert any(f"dbname={verifier.LEGACY_DATABASE}" in part for part in legacy_catalog_call) + legacy_denial = next( + check for check in receipt["checks"]["negative_permissions"] if check["check"] == "legacy_teleo_restore_read" + ) + assert legacy_denial == { + "check": "legacy_teleo_restore_read", + "expected_sqlstate": "42501", + "observed_sqlstate": "42501", + "result": "denied", + } + + +def test_required_sqlstate_mismatch_fails_closed_without_secret_in_error() -> None: + runner = FakeRunner(sqlstate_override=("canonical_insert", "23505")) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + error = caught.value + assert error.code == "negative_permission_sqlstate_mismatch" + assert error.check == "canonical_insert" + assert error.expected_sqlstate == "42501" + assert error.observed_sqlstate == "23505" + assert SECRET_VALUE not in str(error) + assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, error)) + assert_child_boundaries(runner) + + +def test_missing_verbose_sqlstate_fails_closed_without_raw_stderr() -> None: + runner = FakeRunner(sqlstate_missing_for="forged_proposer_overload") + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + error = caught.value + assert error.code == "negative_permission_sqlstate_missing" + assert error.expected_sqlstate == "42883" + assert error.observed_sqlstate == "missing" + assert SECRET_VALUE not in json.dumps(error.as_dict()) + + +def test_unexpected_permission_success_fails_closed_and_transaction_sql_has_rollback() -> None: + runner = FakeRunner(unexpected_success_for="proposal_update") + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "negative_permission_unexpectedly_succeeded" + assert caught.value.observed_sqlstate == "success" + proposal_update = next( + check for check in verifier._negative_checks(RUN_ID, SOURCE_REF) if check.name == "proposal_update" + ) + assert proposal_update.sql.startswith("begin;\n") + assert proposal_update.sql.endswith("\nrollback;") + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize("administrator_mode", ["success", "wrong-error"]) +def test_administrator_secret_must_be_exact_iam_denial_and_stdout_is_discarded(administrator_mode: str) -> None: + runner = FakeRunner(administrator_mode=administrator_mode) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + expected_code = ( + "administrator_secret_unexpectedly_readable" + if administrator_mode == "success" + else "administrator_secret_denial_not_iam_permission" + ) + assert caught.value.code == expected_code + assert SECRET_VALUE not in str(caught.value) + assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value)) + assert runner.calls[-1][3] is True + + +def test_scoped_secret_failure_never_repeats_command_output() -> None: + runner = FakeRunner(scoped_failure=True) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "scoped_secret_access_failed" + assert SECRET_VALUE not in str(caught.value) + assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value)) + assert len(runner.calls) == 1 + + +@pytest.mark.parametrize( + "invalid", + ["short", "UPPERCASE-RUN", "contains_underscore", "contains.dot", " leading-space", "a" * 65], +) +def test_run_id_validation_is_strict(invalid: str) -> None: + with pytest.raises(ValueError, match="8-64 lowercase"): + verifier.validate_run_id(invalid) + + +def test_wrong_identity_or_non_ssl_connection_fails_before_permission_probes() -> None: + runner = FakeRunner(identity_override={"ssl": False}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "private_ssl_identity_mismatch" + assert len(runner.calls) == 2 + assert SECRET_VALUE not in str(caught.value) + + +def test_optional_receipt_is_canonical_mode_0600_and_refuses_symlink(tmp_path: Path) -> None: + payload = {"z": 2, "a": {"safe": True}} + output = tmp_path / "receipt.json" + + verifier.write_receipt(output, payload) + + assert output.read_text(encoding="utf-8") == verifier.canonical_json(payload) + assert stat.S_IMODE(output.stat().st_mode) == 0o600 + assert output.read_text(encoding="utf-8").startswith('{"a":') + + target = tmp_path / "target.json" + target.write_text("untouched", encoding="utf-8") + symlink = tmp_path / "symlink.json" + symlink.symlink_to(target) + with pytest.raises(OSError): + verifier.write_receipt(symlink, payload) + assert target.read_text(encoding="utf-8") == "untouched" + + +def test_subprocess_runner_discards_administrator_stdout_at_file_descriptor(monkeypatch: pytest.MonkeyPatch) -> None: + observed: dict[str, object] = {} + + def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + observed["command"] = command + observed.update(kwargs) + return subprocess.CompletedProcess(command, 1, stdout=None, stderr="PERMISSION_DENIED") + + monkeypatch.setattr(verifier.subprocess, "run", fake_run) + + result = verifier.subprocess_runner([verifier.GCLOUD_BIN, "test"], os.environ, 3, True) + + assert observed["stdout"] is subprocess.DEVNULL + assert observed["stdin"] is subprocess.DEVNULL + assert observed["stderr"] is subprocess.PIPE + assert result.stdout == "" + assert result.returncode == 1 diff --git a/tests/test_gcp_leoclean_service_environment.py b/tests/test_gcp_leoclean_service_environment.py new file mode 100644 index 0000000..a1be9e6 --- /dev/null +++ b/tests/test_gcp_leoclean_service_environment.py @@ -0,0 +1,281 @@ +import json +import os +import subprocess +import sys +from collections.abc import Mapping +from pathlib import Path + +import pytest + +from ops import verify_gcp_leoclean_service_environment as verifier + + +def environment_block(environment: Mapping[str, str]) -> bytes: + return b"\0".join(f"{key}={value}".encode() for key, value in environment.items()) + b"\0" + + +def test_all_expected_fields_pass_and_receipt_contains_no_values() -> None: + environment = verifier.expected_environment() + environment.update({"HOME": "/home/teleo", "LANG": "C.UTF-8"}) + + parsed = verifier.parse_environment(environment_block(environment)) + validated = verifier.validate_environment(parsed) + receipt = { + "runtime_environment": "scoped", + "status": "pass", + "validated_fields": list(validated), + } + rendered = verifier.canonical_json(receipt) + + assert validated == tuple(sorted(verifier.expected_environment())) + assert json.loads(rendered) == receipt + assert rendered == json.dumps(receipt, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n" + for value in verifier.expected_environment().values(): + assert value not in rendered + + +@pytest.mark.parametrize("field", sorted(verifier.expected_environment())) +def test_each_expected_field_mismatch_fails_closed(field: str) -> None: + environment = verifier.expected_environment() + environment[field] = "wrong-value-that-is-not-a-secret" + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + assert caught.value.code == "environment_mismatch" + assert caught.value.fields == (field,) + + +@pytest.mark.parametrize( + ("field", "unsafe_value", "expected_code"), + [ + ("TELEO_CLOUDSQL_USER", "postgres", "environment_mismatch"), + ( + "TELEO_CLOUDSQL_PASSWORD_SECRET", + verifier.ADMINISTRATOR_PASSWORD_SECRET, + "administrator_secret_reference", + ), + ], +) +def test_later_dropin_effective_override_fails_without_outputting_value( + field: str, + unsafe_value: str, + expected_code: str, +) -> None: + environment = verifier.expected_environment() + environment[field] = unsafe_value + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert caught.value.code == expected_code + assert unsafe_value not in rendered + + +@pytest.mark.parametrize( + ("field", "safe_label"), + [ + ("PGPASSWORD", "PG*"), + ("pgservice", "PG*"), + ("CLOUDSDK_CONFIG", "CLOUDSDK_*"), + ("CLOUDSDK_CORE_PROJECT", "CLOUDSDK_*"), + ("CLOUDSDK_AUTH_ACCESS_TOKEN", "CLOUDSDK_*"), + ("cloudsdk_auth_refresh_token", "CLOUDSDK_*"), + ("GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_APPLICATION_CREDENTIALS"), + ("LD_PRELOAD", "LD_*"), + ("LD_LIBRARY_PATH", "LD_*"), + ("ld_audit", "LD_*"), + ("BASH_ENV", "BASH_ENV"), + ("ENV", "ENV"), + ("BASH_FUNC_injected%%", "BASH_FUNC_*"), + ("PYTHONHOME", "PYTHON*"), + ("PYTHONPATH", "PYTHON*"), + ("PYTHONHTTPSVERIFY", "PYTHON*"), + ("pythonstartup", "PYTHON*"), + ("HTTP_PROXY", "HTTP_PROXY"), + ("https_proxy", "HTTPS_PROXY"), + ("ALL_PROXY", "ALL_PROXY"), + ("no_proxy", "NO_PROXY"), + ("SSL_CERT_FILE", "SSL_CERT_FILE"), + ("ssl_cert_dir", "SSL_CERT_DIR"), + ("REQUESTS_CA_BUNDLE", "REQUESTS_CA_BUNDLE"), + ("curl_ca_bundle", "CURL_CA_BUNDLE"), + ("SSLKEYLOGFILE", "SSLKEYLOGFILE"), + ], +) +def test_environment_file_broader_credentials_and_forbidden_fields_fail_closed( + field: str, + safe_label: str, +) -> None: + environment = verifier.expected_environment() + injected_value = "sensitive-injected-value" + environment[field] = injected_value + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert caught.value.code == "forbidden_environment_fields" + assert caught.value.fields == (safe_label,) + assert injected_value not in rendered + + +def test_arbitrary_forbidden_key_suffix_is_not_an_output_channel() -> None: + environment = verifier.expected_environment() + environment["PG_SECRET_MARKER"] = "not-output" + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert "PG_SECRET_MARKER" not in rendered + assert "PG*" in rendered + + +def test_pythonunbuffered_is_the_only_allowed_python_runtime_tuning() -> None: + environment = verifier.expected_environment() + environment["PYTHONUNBUFFERED"] = "1" + + assert verifier.validate_environment(environment) == tuple(sorted(verifier.expected_environment())) + + +def test_bash_env_executes_before_a_script_but_effective_verifier_rejects_it(tmp_path: Path) -> None: + marker = "BASH_ENV_EXECUTED" + payload = tmp_path / "bash-env.sh" + payload.write_text(f"printf '{marker}\\n'\n", encoding="utf-8") + completed = subprocess.run( + ["/bin/bash", "-c", ":"], + env={"BASH_ENV": str(payload), "PATH": verifier.RUNTIME_PATH}, + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 0 + assert completed.stdout.strip() == marker + + environment = verifier.expected_environment() + environment["BASH_ENV"] = str(payload) + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + assert caught.value.fields == ("BASH_ENV",) + + +def test_admin_secret_reference_in_any_value_fails_without_field_or_value_leak() -> None: + environment = verifier.expected_environment() + environment["UNRELATED"] = f"prefix/{verifier.ADMINISTRATOR_PASSWORD_SECRET}/suffix" + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert caught.value.code == "administrator_secret_reference" + assert verifier.ADMINISTRATOR_PASSWORD_SECRET not in rendered + assert "UNRELATED" not in rendered + + +@pytest.mark.parametrize( + "raw", + [ + b"A=1", + b"A=1\0\0", + b"A=1\0BROKEN\0", + b"=value\0", + b"A=1\0A=2\0", + b"A=\xff\0", + b"", + ], +) +def test_malformed_nul_environment_is_rejected(raw: bytes) -> None: + with pytest.raises(verifier.VerificationError) as caught: + verifier.parse_environment(raw) + + assert caught.value.code == "environment_malformed" + + +def test_parser_preserves_equals_inside_values() -> None: + assert verifier.parse_environment(b"A=one=two\0B=\0") == {"A": "one=two", "B": ""} + + +@pytest.mark.parametrize("value", [0, -1, 1.5, "0", "01", "+1", "-3", "not-a-pid", True]) +def test_pid_must_be_a_positive_integer(value: object) -> None: + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_pid(value) # type: ignore[arg-type] + + assert caught.value.code == "invalid_pid" + + +def test_cli_errors_are_canonical_sanitized_json_and_exit_one(capsys: pytest.CaptureFixture[str]) -> None: + secret_input = "not-a-pid-secret-marker" + + returncode = verifier.main(["--pid", secret_input]) + + output = capsys.readouterr() + assert returncode == 1 + assert output.err == "" + assert json.loads(output.out) == {"error": {"code": "invalid_pid"}, "status": "fail"} + assert secret_input not in output.out + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="requires Linux /proc/PID/environ") +def test_cli_reads_real_process_environment_and_detects_effective_conflict( + capsys: pytest.CaptureFixture[str], +) -> None: + expected = verifier.expected_environment() + good_process = subprocess.Popen( + ["/bin/sleep", "30"], + env=expected, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + assert verifier.main(["--pid", str(good_process.pid)]) == 0 + good_output = capsys.readouterr() + assert good_output.err == "" + assert json.loads(good_output.out)["status"] == "pass" + finally: + good_process.terminate() + good_process.wait(timeout=5) + + conflicting = dict(expected) + conflicting["TELEO_CLOUDSQL_USER"] = "postgres" + conflicting["GOOGLE_APPLICATION_CREDENTIALS"] = "/tmp/broader-credential.json" + bad_process = subprocess.Popen( + ["/bin/sleep", "30"], + env=conflicting, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + assert verifier.main(["--pid", str(bad_process.pid)]) == 1 + bad_output = capsys.readouterr() + receipt = json.loads(bad_output.out) + assert bad_output.err == "" + assert receipt == { + "error": {"code": "forbidden_environment_fields", "fields": ["GOOGLE_APPLICATION_CREDENTIALS"]}, + "status": "fail", + } + assert "postgres" not in bad_output.out + assert "/tmp/broader-credential.json" not in bad_output.out + finally: + bad_process.terminate() + bad_process.wait(timeout=5) + + +def test_unexpected_read_error_is_generic_and_does_not_echo_path( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + sensitive_marker = "sensitive-proc-read-detail" + + def fail_read(_self: object) -> bytes: + raise OSError(sensitive_marker) + + monkeypatch.setattr(verifier.Path, "read_bytes", fail_read) + + assert verifier.main(["--pid", str(os.getpid())]) == 1 + output = capsys.readouterr() + assert json.loads(output.out) == {"error": {"code": "internal_error"}, "status": "fail"} + assert sensitive_marker not in output.out diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index 169f0c1..3a41541 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import base64 import contextlib import importlib.util import io @@ -64,6 +65,14 @@ def test_cloudsql_wrapper_supports_expected_review_gated_commands() -> None: assert command in cloudsql_text assert "status" in vps_text + assert "SCRIPT_DIR=${SCRIPT_PATH%/*}" in wrapper_text + assert 'SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"' in wrapper_text + assert 'CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py"' in wrapper_text + assert wrapper_text.startswith("#!/bin/bash\n") + assert "exec /usr/bin/python3" in wrapper_text + assert "command -v psql" in wrapper_text + assert "gcloud" not in wrapper_text + assert '"gcloud"' not in cloudsql_text assert "kb_stage.kb_proposals" in cloudsql_text assert "canonical apply" in cloudsql_text.lower() @@ -95,7 +104,7 @@ def _install_wrapper_fixture(tmp_path: Path, *, missing: frozenset[str] = frozen target = shutil.which(name) assert target (fake_bin / name).symlink_to(target) - for name in ("psql", "gcloud"): + for name in ("psql",): if name in missing: continue executable = fake_bin / name @@ -146,7 +155,7 @@ def test_cloudsql_wrapper_auto_mode_never_falls_back_after_supported_command_fai assert "local" not in completed.stdout -@pytest.mark.parametrize("missing", ["cloudsql-tool", "psql", "gcloud"]) +@pytest.mark.parametrize("missing", ["cloudsql-tool", "psql"]) def test_cloudsql_wrapper_auto_mode_fails_closed_when_prerequisite_is_missing(tmp_path: Path, missing: str) -> None: wrapper, env = _install_wrapper_fixture(tmp_path, missing=frozenset({missing})) @@ -209,6 +218,72 @@ def test_bridge_source_does_not_commit_raw_secret_values() -> None: assert not re.search(pattern, combined), pattern +def _runtime_connection_args(module, **overrides): + values = { + "credential_mode": "runtime", + "user": module.RUNTIME_DB_USER, + "password_secret": module.RUNTIME_PASSWORD_SECRET, + "project": module.RUNTIME_GCP_PROJECT, + "host": module.RUNTIME_CLOUDSQL_HOST, + "port": module.RUNTIME_CLOUDSQL_PORT, + "db": module.RUNTIME_CLOUDSQL_DB, + "canonical_db": module.RUNTIME_CLOUDSQL_DB, + "command": "status", + } + values.update(overrides) + return SimpleNamespace(**values) + + +def _configure_runtime_environment(monkeypatch: pytest.MonkeyPatch) -> None: + for key in tuple(os.environ): + normalized_key = key.upper() + if ( + normalized_key.startswith("PG") + or normalized_key.startswith("CLOUDSDK_") + or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS" + or normalized_key in {"LD_PRELOAD", "PYTHONHOME", "PYTHONPATH"} + or normalized_key == "PROXY" + or normalized_key.endswith("_PROXY") + or normalized_key in { + "CURL_CA_BUNDLE", + "PYTHONHTTPSVERIFY", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SSLKEYLOGFILE", + } + ): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("TELEO_GCP_METADATA_ONLY", "1") + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") + + +class _FakeHttpResponse: + def __init__(self, body: bytes, status: int = 200) -> None: + self.body = body + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + def getcode(self) -> int: + return self.status + + def read(self, _limit: int) -> bytes: + return self.body + + +def _runtime_http_bodies(module, password: bytes = b"scoped-runtime-password") -> list[bytes]: + return [ + module.RUNTIME_SERVICE_ACCOUNT.encode("ascii"), + json.dumps({"access_token": "metadata-access-token", "token_type": "Bearer", "expires_in": 300}).encode(), + json.dumps({"payload": {"data": base64.b64encode(password).decode("ascii")}}).encode(), + ] + + def test_cloudsql_runtime_requires_the_exact_scoped_identity(monkeypatch: pytest.MonkeyPatch) -> None: module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") monkeypatch.delenv("PGPASSWORD", raising=False) @@ -249,17 +324,113 @@ def test_cloudsql_runtime_accepts_only_scoped_credentials_and_rejects_inherited_ monkeypatch: pytest.MonkeyPatch, ) -> None: module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") - args = SimpleNamespace( - credential_mode="runtime", - user=module.RUNTIME_DB_USER, - password_secret=module.RUNTIME_PASSWORD_SECRET, - ) - monkeypatch.delenv("PGPASSWORD", raising=False) + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) module.validate_runtime_connection(args) monkeypatch.setenv("PGPASSWORD", "") - with pytest.raises(SystemExit, match="Refusing inherited PGPASSWORD"): + with pytest.raises(SystemExit, match="Refusing inherited credential, proxy, TLS, or PostgreSQL environment"): + module.validate_runtime_connection(args) + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("GOOGLE_APPLICATION_CREDENTIALS", "/tmp/ambient.json"), + ("google_application_credentials", "/tmp/ambient-lowercase.json"), + ("CLOUDSDK_CONFIG", "/tmp/gcloud"), + ("CLOUDSDK_AUTH_ACCESS_TOKEN", "ambient-token"), + ("cloudsdk_core_project", "attacker-project"), + ("cloudsdk_auth_access_token", "ambient-lowercase-token"), + ("PGHOST", "attacker.invalid"), + ("PGOPTIONS", "-c search_path=attacker"), + ("pgservice", "attacker-service"), + ("ld_preload", "/tmp/attacker.so"), + ("LD_LIBRARY_PATH", "/tmp/attacker-library"), + ("Ld_AuDiT", "/tmp/attacker-audit.so"), + ("BASH_ENV", "/tmp/attacker-bash-env"), + ("ENV", "/tmp/attacker-sh-env"), + ("BASH_FUNC_psql%%", "() { :; }"), + ("PythonHome", "/tmp/attacker-python"), + ("pythonpath", "/tmp/attacker-modules"), + ("PYTHONHTTPSVERIFY", "0"), + ("pythonstartup", "/tmp/attacker-startup.py"), + ("https_proxy", "http://attacker.invalid:8080"), + ("HtTp_PrOxY", "http://attacker.invalid:8080"), + ("ssl_cert_file", "/tmp/attacker.pem"), + ("SSL_CERT_DIR", "/tmp/attacker-certs"), + ("REQUESTS_CA_BUNDLE", "/tmp/attacker.pem"), + ("SSLKEYLOGFILE", "/tmp/attacker-tls-keys"), + ], +) +def test_cloudsql_runtime_rejects_ambient_auth_and_pg_environment( + monkeypatch: pytest.MonkeyPatch, + name: str, + value: str, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.setenv(name, value) + + with pytest.raises(SystemExit, match="Refusing inherited credential, proxy, TLS, or PostgreSQL environment"): + module.validate_runtime_connection(args) + + +def test_cloudsql_runtime_allows_only_pythonunbuffered_runtime_tuning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.setenv("PYTHONUNBUFFERED", "1") + + module.validate_runtime_connection(args) + + +def test_cloudsql_runtime_requires_explicit_metadata_only_mode(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.delenv("TELEO_GCP_METADATA_ONLY") + + with pytest.raises(SystemExit, match="TELEO_GCP_METADATA_ONLY=1"): + module.validate_runtime_connection(args) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("project", "other-project", "project=teleo-501523"), + ("host", "127.0.0.1", "host=10.61.0.3"), + ("port", "6543", "port=5432"), + ("db", "teleo_kb", "db=teleo_canonical"), + ("canonical_db", "teleo_kb", "canonical_db=teleo_canonical"), + ], +) +def test_cloudsql_runtime_rejects_any_target_drift_before_credentials( + monkeypatch: pytest.MonkeyPatch, + field: str, + value: str, + message: str, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module, **{field: value}) + monkeypatch.delenv("PGPASSWORD", raising=False) + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") + + with pytest.raises(SystemExit, match=message): + module.validate_runtime_connection(args) + + +def test_cloudsql_runtime_rejects_audit_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + monkeypatch.delenv("PGPASSWORD", raising=False) + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "1") + + with pytest.raises(SystemExit, match="AUDIT_FALLBACK=0"): module.validate_runtime_connection(args) @@ -290,33 +461,44 @@ def test_cloudsql_clone_mode_is_read_only_and_bound_to_one_disposable_database( module.validate_runtime_connection(args) -def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pytest.MonkeyPatch) -> None: +def test_cloudsql_clone_mode_never_resolves_a_password_secret_argument(monkeypatch: pytest.MonkeyPatch) -> None: module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") args = SimpleNamespace( - credential_mode="runtime", - user=module.RUNTIME_DB_USER, - password_secret=module.RUNTIME_PASSWORD_SECRET, - host="10.61.0.3", - port="5432", - db="teleo_canonical", - canonical_db="teleo_canonical", - project="teleo-501523", + credential_mode="clone-readonly", + user="postgres", + password_secret="gcp-teleo-pgvector-standby-postgres-password", + canonical_db="teleo_clone_canary_123", + db="teleo_clone_canary_123", command="status", ) monkeypatch.delenv("PGPASSWORD", raising=False) - monkeypatch.setenv("PGHOST", "attacker.invalid") - monkeypatch.setenv("PGSERVICE", "untrusted-service") - monkeypatch.setenv("PGPASSFILE", "/tmp/untrusted-pgpass") - monkeypatch.setenv("PGOPTIONS", "-c search_path=attacker") + monkeypatch.setenv("PGOPTIONS", module.CLONE_READ_ONLY_PGOPTIONS) + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_args, **_kwargs: pytest.fail("a password-secret argument must never launch a credential helper"), + ) + + with pytest.raises(SystemExit, match="requires an explicit user and inherited test credential"): + module.password(args) + + +def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.setenv("HOME", "/tmp/untrusted-home") + monkeypatch.setenv("LANG", "attacker-locale") + monkeypatch.setenv("SHELL", "/tmp/untrusted-shell") + monkeypatch.setenv("UNRELATED_AMBIENT_VALUE", "must-not-reach-psql") captured: dict[str, object] = {} def fake_run(command, **kwargs): - if command[0] == "gcloud": - return SimpleNamespace(returncode=0, stdout="scoped-runtime-password\n", stderr="") captured["command"] = command captured["kwargs"] = kwargs return SimpleNamespace(returncode=0, stdout="ok\n", stderr="") + monkeypatch.setattr(module, "runtime_password_from_metadata", lambda: "scoped-runtime-password") monkeypatch.setattr(module.subprocess, "run", fake_run) assert module.run_psql(args, "select 1;") == "ok\n" @@ -324,16 +506,279 @@ def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pyt kwargs = captured["kwargs"] assert isinstance(kwargs, dict) env = kwargs["env"] - assert env["PGHOST"] == "10.61.0.3" - assert env["PGPORT"] == "5432" - assert env["PGDATABASE"] == "teleo_canonical" - assert env["PGUSER"] == module.RUNTIME_DB_USER - assert env["PGSSLMODE"] == "require" - assert env["PGCONNECT_TIMEOUT"] == "8" - assert env["PGPASSWORD"] == "scoped-runtime-password" - assert "PGOPTIONS" not in env - assert "PGSERVICE" not in env - assert "PGPASSFILE" not in env + assert env == { + "PATH": "/usr/bin:/bin", + "PGHOST": "10.61.0.3", + "PGPORT": "5432", + "PGDATABASE": "teleo_canonical", + "PGUSER": module.RUNTIME_DB_USER, + "PGSSLMODE": "require", + "PGCONNECT_TIMEOUT": "8", + "PGPASSWORD": "scoped-runtime-password", + } + + +def test_cloudsql_runtime_metadata_and_secret_requests_are_exact_and_silent( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + bodies = _runtime_http_bodies(module) + requests: list[tuple[str, str, dict[str, str], float]] = [] + + def fake_urlopen(request, *, timeout): + requests.append( + ( + request.full_url, + request.get_method(), + {key.lower(): value for key, value in request.header_items()}, + timeout, + ) + ) + return _FakeHttpResponse(bodies.pop(0)) + + monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", fake_urlopen) + + assert module.password(args) == "scoped-runtime-password" + assert module.GCE_METADATA_EMAIL_URL == ( + "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" + ) + assert module.GCE_METADATA_TOKEN_URL == ( + "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" + ) + assert requests == [ + ( + module.GCE_METADATA_EMAIL_URL, + "GET", + {"metadata-flavor": "Google"}, + module.HTTP_TIMEOUT_SECONDS, + ), + ( + module.GCE_METADATA_TOKEN_URL, + "GET", + {"metadata-flavor": "Google"}, + module.HTTP_TIMEOUT_SECONDS, + ), + ( + module.RUNTIME_SECRET_ACCESS_URL, + "GET", + {"accept": "application/json", "authorization": "Bearer metadata-access-token"}, + module.HTTP_TIMEOUT_SECONDS, + ), + ] + assert capsys.readouterr() == ("", "") + + +def test_cloudsql_runtime_http_opener_disables_proxies_and_redirects() -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + proxy_handlers = [ + handler for handler in module.RUNTIME_HTTP_OPENER.handlers if isinstance(handler, module.urllib.request.ProxyHandler) + ] + redirect_handlers = [ + handler + for handler in module.RUNTIME_HTTP_OPENER.handlers + if isinstance(handler, module.urllib.request.HTTPRedirectHandler) + ] + + # An empty ProxyHandler suppresses build_opener's environment-backed default; + # urllib omits the empty handler itself from the final chain. + assert proxy_handlers == [] + assert module.RUNTIME_PROXY_HANDLER.proxies == {} + assert len(redirect_handlers) == 1 + assert type(redirect_handlers[0]) is module._RejectRedirectHandler + + +def test_cloudsql_runtime_rejects_wrong_metadata_service_account_before_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + calls: list[str] = [] + + def fake_urlopen(request, *, timeout): + assert timeout == module.HTTP_TIMEOUT_SECONDS + calls.append(request.full_url) + return _FakeHttpResponse(b"unexpected-sa@teleo-501523.iam.gserviceaccount.com") + + monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", fake_urlopen) + + with pytest.raises(SystemExit, match="did not match the required service account"): + module.password(args) + assert calls == [module.GCE_METADATA_EMAIL_URL] + + +@pytest.mark.parametrize( + "token_body", + [ + b"not-json", + b"[]", + b'{"access_token":"token","token_type":"Basic","expires_in":300}', + b'{"access_token":"token with spaces","token_type":"Bearer","expires_in":300}', + b'{"access_token":"token","token_type":"Bearer","expires_in":0}', + b'{"access_token":"one","access_token":"two","token_type":"Bearer","expires_in":300}', + ], +) +def test_cloudsql_runtime_rejects_malformed_metadata_tokens_without_leaking_them( + monkeypatch: pytest.MonkeyPatch, + token_body: bytes, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + bodies = [module.RUNTIME_SERVICE_ACCOUNT.encode("ascii"), token_body] + monkeypatch.setattr( + module.RUNTIME_HTTP_OPENER, + "open", + lambda _request, *, timeout: _FakeHttpResponse(bodies.pop(0)), + ) + + with pytest.raises(SystemExit, match="Runtime metadata token response was invalid") as captured: + module.password(args) + assert "token with spaces" not in str(captured.value) + + +@pytest.mark.parametrize( + "secret_body", + [ + b"not-json", + b"[]", + b"{}", + b'{"payload":{"data":"%%%"}}', + json.dumps({"payload": {"data": base64.b64encode(b"x" * 4097).decode("ascii")}}).encode(), + json.dumps({"payload": {"data": base64.b64encode(b"line1\nline2").decode("ascii")}}).encode(), + json.dumps({"payload": {"data": base64.b64encode(b"line1\rline2").decode("ascii")}}).encode(), + json.dumps({"payload": {"data": base64.b64encode(b"nul\x00byte").decode("ascii")}}).encode(), + ], +) +def test_cloudsql_runtime_rejects_malformed_secret_payloads_without_leaking_them( + monkeypatch: pytest.MonkeyPatch, + secret_body: bytes, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + bodies = _runtime_http_bodies(module) + bodies[-1] = secret_body + monkeypatch.setattr( + module.RUNTIME_HTTP_OPENER, + "open", + lambda _request, *, timeout: _FakeHttpResponse(bodies.pop(0)), + ) + + with pytest.raises(SystemExit, match=r"Scoped runtime secret .* invalid") as captured: + module.password(args) + assert secret_body.decode("utf-8", errors="ignore") not in str(captured.value) + + +def test_cloudsql_runtime_admin_secret_is_impossible_through_arguments( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module, password_secret="gcp-teleo-pgvector-standby-postgres-password") + _configure_runtime_environment(monkeypatch) + monkeypatch.setattr( + module.RUNTIME_HTTP_OPENER, + "open", + lambda *_args, **_kwargs: pytest.fail("metadata must not be queried for an admin secret argument"), + ) + + with pytest.raises(SystemExit, match="requires scoped password secret"): + module.password(args) + + +def test_cloudsql_runtime_http_and_psql_failures_never_echo_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + dummy = "DUMMY-RUNTIME-SECRET-DO-NOT-LEAK" + + def failed_urlopen(_request, *, timeout): + raise RuntimeError(f"provider echoed {dummy}") + + monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", failed_urlopen) + with pytest.raises(SystemExit) as http_error: + module.password(args) + assert dummy not in str(http_error.value) + + monkeypatch.setattr(module, "runtime_password_from_metadata", lambda: dummy) + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=2, stdout=dummy, stderr=dummy), + ) + with pytest.raises(SystemExit, match="Cloud SQL query failed with exit status 2") as psql_error: + module.run_psql(args, "select 1;") + assert dummy not in str(psql_error.value) + + +def test_cloudsql_runtime_read_marker_binds_database_user_private_server_and_ssl( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + marker = { + "database": module.RUNTIME_CLOUDSQL_DB, + "database_user": module.RUNTIME_DB_USER, + "server_addr": module.RUNTIME_CLOUDSQL_HOST, + "server_port": int(module.RUNTIME_CLOUDSQL_PORT), + "ssl": True, + "ssl_version": "TLSv1.3", + "system_identifier": "system-id", + "wal_lsn": "0/1", + } + captured: dict[str, object] = {} + + def fake_psql_json_lines(_args, sql, db=None): + captured["sql"] = sql + captured["db"] = db + return [marker] + + monkeypatch.setattr(module, "psql_json_lines", fake_psql_json_lines) + + assert module.database_read_marker(args) == marker + assert captured["db"] == module.RUNTIME_CLOUDSQL_DB + assert "inet_server_addr()" in str(captured["sql"]) + assert "pg_stat_ssl" in str(captured["sql"]) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("database", "teleo_kb", "unexpected database identity"), + ("database_user", "postgres", "unexpected database identity"), + ("server_addr", "127.0.0.1", "unexpected server address"), + ("server_port", 6543, "unexpected server port"), + ("ssl", False, "encrypted connection"), + ], +) +def test_cloudsql_runtime_read_marker_rejects_wrong_live_identity( + monkeypatch: pytest.MonkeyPatch, + field: str, + value: object, + message: str, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + marker = { + "database": module.RUNTIME_CLOUDSQL_DB, + "database_user": module.RUNTIME_DB_USER, + "server_addr": module.RUNTIME_CLOUDSQL_HOST, + "server_port": int(module.RUNTIME_CLOUDSQL_PORT), + "ssl": True, + "ssl_version": "TLSv1.3", + "system_identifier": "system-id", + "wal_lsn": "0/1", + } + marker[field] = value + monkeypatch.setattr(module, "psql_json_lines", lambda *_args, **_kwargs: [marker]) + + with pytest.raises(SystemExit, match=message): + module.database_read_marker(args) def test_cloudsql_proposal_calls_leave_leo_identity_to_the_database(monkeypatch: pytest.MonkeyPatch) -> None: @@ -390,6 +835,7 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None: assert "can_insert_proposals_directly" in sql assert "can_create_database_objects" in sql assert "can_create_temp_objects" in sql + assert "can_connect_legacy_database" in sql assert "unexpectedly has database CREATE" in sql assert "REVOKE TEMP FROM PUBLIC would be a" in sql assert "can execute unexpected SECURITY DEFINER functions" in sql @@ -407,6 +853,15 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None: ) assert "grant update" not in sql.lower() assert "grant delete" not in sql.lower() + assert "create role leoclean_kb_runtime nologin" in sql.lower() + disable_statement = "alter role leoclean_kb_runtime\n with nologin nosuperuser" + password_statement = "alter role leoclean_kb_runtime password" + assert sql.lower().index(disable_statement) < sql.lower().index(password_statement) + assert sql.lower().index(disable_statement) < sql.index("\\getenv runtime_password") + assert sql.index("\\getenv runtime_password") < sql.lower().index(password_statement) + assert sql.lower().index("with nologin nosuperuser") < sql.index("\\connect teleo_kb") + assert sql.index("\\connect teleo_kb") < sql.lower().rindex("with login nosuperuser") + assert "leoclean_kb_runtime final login attributes are unsafe" in sql def _load_module(path: Path): diff --git a/tests/test_teleo_agent_systemd.py b/tests/test_teleo_agent_systemd.py index 99c0535..f6edaaa 100644 --- a/tests/test_teleo_agent_systemd.py +++ b/tests/test_teleo_agent_systemd.py @@ -187,15 +187,60 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert 'DROPIN_NAME="10-cloudsql-memory.conf"' in script assert "20-canonical-kb.conf" not in script assert "sha256sum" in script + assert "--preflight-only" in script assert "--restart" in script - assert 'python3 "$tool_path" status --format json --redacted' in script + assert '/usr/bin/python3 "$tool_path" status --format json --redacted' in script assert "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" in script - assert "metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" in script - assert "env -u PGPASSWORD" in script - assert 'run_scoped_status pre "$remote_tmp/$TOOL_REL"' in script - assert 'run_scoped_status post "$profile_bin/cloudsql_memory_tool.py"' in script + assert "169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" in script + assert "ProxyHandler({})" in script + assert "env -i" in script + assert 'run_scoped_status pre "$payload_dir/$TOOL_REL"' in script + assert 'run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL"' in script + assert 'sudo -n -u teleo mkdir -p -- "$receipt_dir"' in script + assert 'if [ "$PREFLIGHT_ONLY" = true ]; then' in script + assert "run_service_wrapper_status post" in script + assert "run_service_wrapper_status verify" in script + assert '"$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted' in script + wrapper_probe = script.split("run_service_wrapper_status() {", maxsplit=1)[1].split( + "\n}\n\nservice_fingerprint", maxsplit=1 + )[0] + assert 'nsenter -t "$main_pid" -m --env' in wrapper_probe + assert "/usr/bin/setpriv" in wrapper_probe + assert "--reuid=teleo" in wrapper_probe + assert "TELEO_CLOUDSQL_" not in wrapper_probe + assert "/usr/bin/env -i" not in wrapper_probe + assert 'run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py"' in script + assert 'SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py"' in script + assert 'assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py"' in script + assert 'assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"' in script + assert "REMOTE_CLOUDSDK_CONFIG" not in script + assert "--property=DropInPaths" in script + assert "--property=EnvironmentFiles" in script + assert "Service has an effective EnvironmentFile" in script + assert "Service has a later drop-in" in script + assert "assert_administrator_secret_denied" in script + assert '"stdout_discarded": True' in script + assert '"credential_context": "systemd_main_pid_environment"' in script + administrator_probe = script.split("assert_administrator_secret_denied() {", maxsplit=1)[1].split( + "\n}\n\nrun_scoped_status", maxsplit=1 + )[0] + assert 'nsenter -t "$main_pid" -m --env' in administrator_probe + assert "/usr/bin/setpriv" in administrator_probe + assert "env -i" not in administrator_probe + assert 'preflight_service_before="$(service_fingerprint)"' in script + assert 'assert_service_fingerprint "$preflight_service_before"' in script assert "backup_path" in script + assert "backup_directory_state" in script assert "restore_path" in script + assert "restore_directory_state" in script + assert 'runtime_grandparent="$(dirname "$runtime_parent")"' in script + assert 'runtime_anchor="$(dirname "$runtime_grandparent")"' in script + assert 'dropin_parent="$(dirname "$dropin_dir")"' in script + assert 'backup_directory_state "$runtime_grandparent" runtime_grandparent' in script + assert 'restore_directory_state "$runtime_grandparent" runtime_grandparent' in script + assert 'sudo test -d "$runtime_anchor"' in script + assert 'sudo test -d "$dropin_parent"' in script + assert 'sudo rmdir -- "$target"' in script assert "rollback_runtime" in script assert "trap 'on_exit $?' EXIT" in script assert 'backup_path "$dropin_dir/$DROPIN_NAME" dropin' in script @@ -205,13 +250,303 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert 'sub_state" != "running"' in script assert 'sudo systemctl stop "$SERVICE"' in script assert 'sudo systemctl start "$SERVICE"' in script + assert 'REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"' in script + assert 'sudo install -d -o root -g root -m 0755 "$runtime_parent"' in script + assert 'sudo install -d -o root -g root -m 0755 "$runtime_bin"' in script + assert 'remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)"' in script + assert 'sudo install -d -o root -g root -m 0755 "$payload_dir"' in script + assert 'sudo install -d -o root -g root -m 0700 "$backup_dir"' in script + assert 'sudo install -d -o teleo -g teleo -m 0700 "$work_dir"' in script + assert "sudo tar --no-same-owner --no-same-permissions" in script + assert 'sudo chown -R root:root "$payload_dir"' in script + assert 'sudo find "$payload_dir" -type d -exec chmod 0555 {} +' in script + assert 'sudo find "$payload_dir" -type l -print -quit' in script + assert 'sudo -n -u teleo test ! -w "$payload_dir"' in script + assert 'sudo -n -u teleo test ! -w "$backup_dir"' in script + assert 'sudo touch "$backup_dir/$name.present"' in script + assert 'sudo test -f "$backup_dir/$name.present"' in script + assert 'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage"' in script + assert 'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage"' in script + assert ( + 'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" ' + '"$permission_verifier_stage"' in script + ) + assert ( + 'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" ' + '"$service_env_verifier_stage"' in script + ) + assert 'assert_installed_files\nsudo systemctl daemon-reload' in script + assert 'tar -C "$remote_tmp" -xf -' not in script + assert 'mkdir -p "$backup_dir"' not in script + assert 'for protected_file in \\\n "$wrapper" \\' in script + assert '"$permission_verifier" \\' in script + assert '"$service_env_verifier" \\' in script + assert 'sudo -n -u teleo test ! -w "$protected_file"' in script + assert 'for protected_dir in \\\n /usr \\' in script + assert "/usr/local/libexec/livingip" in script + assert 'sudo -n -u teleo test ! -w "$protected_dir"' in script + assert 'for executable_dir in \\\n /usr/local/sbin \\' in script + assert 'sudo stat -Lc \'%U:%G\' "$executable_dir"' in script + assert 'sudo -n -u teleo test ! -w "$executable_dir"' in script + assert 'for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3' in script + assert '/usr/bin/curl -q --noproxy' in script + assert 'test "$(sudo cat "$revision")" = "$SOURCE_REVISION"' in script + assert 'assert_existing_root_directory "$existing_directory"' in script + assert 'assert_existing_root_file "$existing_file"' in script + assert 'validate_restored_path "$runtime_bin/teleo-kb" wrapper' in script + assert 'validate_restored_directory "$runtime_bin" runtime_bin' in script + rollback = script.split("rollback_runtime() {", maxsplit=1)[1].split("\n}\n\non_exit()", maxsplit=1)[0] + assert rollback.index("validate_restored_path") < rollback.index("systemctl daemon-reload") + assert rollback.index("systemctl daemon-reload") < rollback.index('systemctl start "$SERVICE"') + assert rollback.count('if [ "$rollback_rc" -eq 0 ]; then') >= 3 + deploy_tail = script.split("mutation_started=true", maxsplit=1)[1] + assert deploy_tail.index("assert_installed_files") < deploy_tail.index("systemctl daemon-reload") + assert deploy_tail.index("systemctl daemon-reload") < deploy_tail.index("assert_unit_configuration") + assert deploy_tail.index("assert_unit_configuration") < deploy_tail.index('systemctl start "$SERVICE"') + for install_line in ( + 'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL"', + 'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL"', + 'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL"', + 'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL"', + 'sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL"', + ): + assert install_line in deploy_tail + assert 'sudo install -o root -g root -m 0755 "$work_dir/' not in script + assert 'nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb"' in script + assert 'findmnt -T "$SERVICE_PROFILE_BIN"' in script + assert 'findmnt -M "$SERVICE_HERMES_ROOT"' in script + assert 'findmnt -M "$SERVICE_PROFILE_ROOT/state"' in script + assert 'findmnt -M "$SERVICE_PROFILE_ROOT/workspace"' in script + assert "cap_sys_admin" in script + assert '"current_tier": "T3_live_readonly"' in script + assert '"status": "pass"' in script assert "UnsetEnvironment=PGPASSWORD" in dropin - assert "RuntimeDirectory=leoclean-gcp-prod-parallel" in dropin - assert "RuntimeDirectoryMode=0700" in dropin - assert "Environment=CLOUDSDK_CONFIG=/run/leoclean-gcp-prod-parallel" in dropin + assert "UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG" in dropin + assert "UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS" in dropin + assert "UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT" in dropin + assert "UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT" in dropin + assert "UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY" in dropin + assert "UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE" in dropin + assert ( + "BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin" in dropin + ) + assert "ReadOnlyPaths=/home/teleo/.hermes" in dropin + assert ( + "ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state " + "/home/teleo/.hermes/profiles/leoclean/workspace" in dropin + ) + assert "CapabilityBoundingSet=~CAP_SYS_ADMIN" in dropin + assert "RuntimeDirectory=" not in dropin + assert "Environment=CLOUDSDK_" not in dropin + assert "Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" in dropin assert "TELEO_KB_MODE=cloudsql" in dropin + assert "TELEO_GCP_METADATA_ONLY=1" in dropin assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in dropin assert "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" in dropin assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in dropin assert not any(line.startswith("Environment=PGPASSWORD") for line in dropin.splitlines()) assert "gcp-teleo-pgvector-standby-postgres-password" not in dropin + + +def test_gcp_runtime_embedded_remote_programs_are_bash_syntax_valid() -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + blocks: dict[str, str] = {} + + for variable in ("remote_helpers", "verify_body", "sync_body"): + marker = f"{variable}=$(cat <<'REMOTE'\n" + assert script.count(marker) == 1 + start = script.index(marker) + len(marker) + end = script.index("\nREMOTE\n)", start) + blocks[variable] = script[start:end] + + for name, body in ( + *blocks.items(), + ("composed_verify", blocks["remote_helpers"] + "\n" + blocks["verify_body"]), + ("composed_sync", blocks["remote_helpers"] + "\n" + blocks["sync_body"]), + ): + completed = subprocess.run( + ["bash", "-n"], + input=body, + capture_output=True, + check=False, + text=True, + ) + assert completed.returncode == 0, f"{name}: {completed.stderr}" + + embedded_python = [part.split("\nPY", maxsplit=1)[0] for part in blocks["remote_helpers"].split("<<'PY'\n")[1:]] + assert len(embedded_python) == 2 + for index, body in enumerate(embedded_python): + completed = subprocess.run( + ["python3", "-c", "import sys; compile(sys.stdin.read(), '', 'exec')"], + input=body, + capture_output=True, + check=False, + text=True, + ) + assert completed.returncode == 0, f"embedded_python_{index}: {completed.stderr}" + + +def test_gcp_runtime_rejects_preexisting_symlink_before_mutation(tmp_path: Path) -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("assert_existing_root_directory() {") + function_end = script.index("\n\nbackup_path() {", function_start) + trust_functions = script[function_start:function_end] + real_directory = tmp_path / "real-runtime" + real_directory.mkdir() + symlink = tmp_path / "runtime-link" + symlink.symlink_to(real_directory, target_is_directory=True) + harness = f""" +set -euo pipefail +sudo() {{ + if [ "${{1:-}}" = -n ]; then + shift + if [ "${{1:-}}" = -u ]; then shift 2; fi + fi + "$@" +}} +{trust_functions} +assert_existing_root_directory "$1" +""" + completed = subprocess.run( + ["bash", "-s", "--", str(symlink)], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode != 0 + + +def test_gcp_runtime_executable_configuration_override_gate() -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("assert_unit_configuration() {") + function_end = script.index("\n\nassert_runtime_environment() {", function_start) + unit_configuration_function = script[function_start:function_end] + harness = f""" +set -euo pipefail +SERVICE=leoclean.service +REMOTE_DROPIN_DIR=/etc/systemd/system/leoclean.service.d +DROPIN_NAME=10-cloudsql-memory.conf +systemctl() {{ + case "$*" in + *--property=DropInPaths*) printf '%s\\n' "${{FAKE_DROPINS:-}}" ;; + *--property=EnvironmentFiles*) printf '%s\\n' "${{FAKE_ENVIRONMENT_FILES:-}}" ;; + *) return 90 ;; + esac +}} +{unit_configuration_function} +assert_unit_configuration +""" + reviewed = "/etc/systemd/system/leoclean.service.d/10-cloudsql-memory.conf" + cases = ( + { + "FAKE_DROPINS": f"{reviewed} /etc/systemd/system/leoclean.service.d/99-conflict.conf", + "FAKE_ENVIRONMENT_FILES": "", + }, + {"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": "/etc/leoclean-runtime.env"}, + ) + for environment in cases: + completed = subprocess.run( + ["bash", "-s"], + input=harness, + capture_output=True, + check=False, + env=environment, + text=True, + ) + assert completed.returncode != 0 + + allowed = subprocess.run( + ["bash", "-s"], + input=harness, + capture_output=True, + check=False, + env={"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": ""}, + text=True, + ) + assert allowed.returncode == 0, allowed.stderr + + +def test_gcp_runtime_absence_rollback_removes_every_created_directory(tmp_path: Path) -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("backup_directory_state() {") + function_end = script.index("\nrestore_path() {", function_start) + directory_functions = script[function_start:function_end] + harness = f""" +set -euo pipefail +sudo() {{ "$@"; }} +root="$1" +backup_dir="$root/backup" +mkdir -p "$backup_dir" "$root/usr/local" "$root/etc/systemd/system" +runtime_grandparent="$root/usr/local/libexec" +runtime_parent="$runtime_grandparent/livingip" +runtime_bin="$runtime_parent/leoclean-kb" +dropin_dir="$root/etc/systemd/system/service.d" +{directory_functions} +backup_directory_state "$runtime_grandparent" runtime_grandparent +backup_directory_state "$runtime_parent" runtime_parent +backup_directory_state "$runtime_bin" runtime_bin +backup_directory_state "$dropin_dir" dropin_dir +mkdir -p "$runtime_bin" "$dropin_dir" +restore_directory_state "$runtime_bin" runtime_bin +restore_directory_state "$runtime_parent" runtime_parent +restore_directory_state "$runtime_grandparent" runtime_grandparent +restore_directory_state "$dropin_dir" dropin_dir +test ! -e "$runtime_grandparent" +test ! -e "$dropin_dir" +test -d "$root/usr/local" +test -d "$root/etc/systemd/system" +""" + completed = subprocess.run( + ["bash", "-s", "--", str(tmp_path)], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + + +def test_gcp_runtime_rollback_never_reloads_or_restarts_after_validation_failure(tmp_path: Path) -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("rollback_runtime() {") + function_end = script.index("\n\non_exit() {", function_start) + rollback_function = script[function_start:function_end] + action_log = tmp_path / "systemctl-actions" + harness = f""" +set -u +SERVICE=leoclean.service +runtime_bin=/usr/local/libexec/livingip/leoclean-kb +runtime_parent=/usr/local/libexec/livingip +runtime_grandparent=/usr/local/libexec +dropin_dir=/etc/systemd/system/leoclean.service.d +DROPIN_NAME=10-cloudsql-memory.conf +action_log="$1" +sudo() {{ + if [ "${{1:-}}" = systemctl ]; then + printf '%s\\n' "$2" >>"$action_log" + return 0 + fi + "$@" +}} +restore_path() {{ return 0; }} +restore_directory_state() {{ return 0; }} +validate_restored_path() {{ [ "$2" != wrapper ]; }} +validate_restored_directory() {{ return 0; }} +assert_unit_configuration() {{ printf '%s\\n' unit-configuration >>"$action_log"; }} +assert_service_running() {{ printf '%s\\n' assert-running >>"$action_log"; }} +{rollback_function} +rollback_runtime +""" + completed = subprocess.run( + ["bash", "-s", "--", str(action_log)], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode != 0 + assert action_log.read_text(encoding="utf-8").splitlines() == ["stop"]