From ce90a79aaccb643882dac335fa9559170760e16c Mon Sep 17 00:00:00 2001 From: fwazb Date: Tue, 14 Jul 2026 12:57:31 -0700 Subject: [PATCH 1/7] fix: scope GCP Leo database runtime --- deploy/sync-gcp-leoclean-runtime.sh | 361 ++++++++ docs/gcp-leoclean-runtime-reconciliation.md | 157 ++++ ...-db-first-live-deploy-restart-current.json | 7 +- .../gcp-db-first-working-leo-20260714.md | 19 +- .../leoclean-bin/cloudsql_memory_tool.py | 183 ++-- hermes-agent/leoclean-bin/teleo-kb | 29 +- ops/gcp_leoclean_runtime_role.sql | 847 ++++++++++++++++++ ...run_gcp_generated_db_direct_claim_suite.py | 4 +- .../leoclean-gcp-prod-parallel-cloudsql.conf | 14 + ...est_gcp_generated_db_direct_claim_suite.py | 4 + .../test_hermes_leoclean_kb_bridge_source.py | 286 +++++- tests/test_teleo_agent_systemd.py | 94 +- 12 files changed, 1867 insertions(+), 138 deletions(-) create mode 100755 deploy/sync-gcp-leoclean-runtime.sh create mode 100644 docs/gcp-leoclean-runtime-reconciliation.md create mode 100644 ops/gcp_leoclean_runtime_role.sql create mode 100644 systemd/leoclean-gcp-prod-parallel-cloudsql.conf diff --git a/deploy/sync-gcp-leoclean-runtime.sh b/deploy/sync-gcp-leoclean-runtime.sh new file mode 100755 index 0000000..3661155 --- /dev/null +++ b/deploy/sync-gcp-leoclean-runtime.sh @@ -0,0 +1,361 @@ +#!/usr/bin/env bash +# Deploy the source-controlled Cloud SQL bridge and non-secret service settings +# to the GCP parallel leoclean runtime. This does not create database users, +# read secret values, send Telegram messages, or promote Cloud SQL. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PROJECT="${PROJECT:-teleo-501523}" +ZONE="${ZONE:-europe-west6-a}" +INSTANCE="${INSTANCE:-teleo-prod-1}" +SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}" +MERGED_REF="${MERGED_REF:-origin/main}" +REMOTE_PROFILE_BIN="${REMOTE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}" +REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}" +REMOTE_CLOUDSDK_CONFIG="${REMOTE_CLOUDSDK_CONFIG:-/run/leoclean-gcp-prod-parallel}" +EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}" + +WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb" +TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py" +DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf" +DROPIN_NAME="10-cloudsql-memory.conf" + +DRY_RUN=false +RESTART=false +VERIFY_ONLY=false + +usage() { + cat <<'USAGE' +Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--restart] [--verify-only] + +The database role and scoped Secret Manager secret must exist before --restart. +This script deploys only the bridge files and non-secret systemd drop-in. +USAGE +} + +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --restart) RESTART=true ;; + --verify-only) VERIFY_ONLY=true ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown arg: $arg" >&2 + usage >&2 + exit 64 + ;; + esac +done + +for path in "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; do + if [ ! -f "$REPO_ROOT/$path" ]; then + echo "Required source file is missing: $path" >&2 + exit 66 + fi +done + +if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2 + exit 65 +fi +if ! grep -Fxq "Environment=CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG" "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in CLOUDSDK_CONFIG does not match $REMOTE_CLOUDSDK_CONFIG." >&2 + exit 65 +fi + +SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)" +if ! git -C "$REPO_ROOT" diff --quiet HEAD -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; then + echo "Refusing to deploy runtime files that do not match HEAD." >&2 + exit 65 +fi +if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL")" ]; then + echo "Refusing to deploy untracked runtime files." >&2 + exit 65 +fi +if ! git -C "$REPO_ROOT" rev-parse --verify "$MERGED_REF^{commit}" >/dev/null 2>&1; then + echo "Merged reference is unavailable: $MERGED_REF" >&2 + exit 69 +fi +if ! git -C "$REPO_ROOT" merge-base --is-ancestor "$SOURCE_REVISION" "$MERGED_REF"; then + echo "Refusing to deploy unmerged revision $SOURCE_REVISION (not contained in $MERGED_REF)." >&2 + exit 65 +fi +WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')" +TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')" +DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')" + +GCP_SSH=( + gcloud compute ssh "$INSTANCE" + --project="$PROJECT" + --zone="$ZONE" + --tunnel-through-iap +) + +remote_helpers=$(cat <<'REMOTE' +assert_service_running() { + local active_state sub_state main_pid unit_user process_user + active_state="$(systemctl show "$SERVICE" --property=ActiveState --value)" + sub_state="$(systemctl show "$SERVICE" --property=SubState --value)" + if [ "$active_state" != "active" ] || [ "$sub_state" != "running" ]; then + echo "Service is not active/running: ActiveState=$active_state SubState=$sub_state" >&2 + return 1 + fi + + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + unit_user="$(systemctl show "$SERVICE" --property=User --value)" + case "$main_pid" in + ''|0|*[!0-9]*) + echo "Service has no live MainPID: $main_pid" >&2 + return 1 + ;; + esac + if [ "$unit_user" != "teleo" ]; then + echo "Service unit user is not teleo: $unit_user" >&2 + return 1 + fi + process_user="$(ps -o user= -p "$main_pid" | awk '{print $1}')" + if [ "$process_user" != "teleo" ]; then + echo "Service process user is not teleo: $process_user" >&2 + return 1 + fi +} + +assert_runtime_environment() { + local environment runtime_dir_state + environment="$(systemctl show "$SERVICE" --property=Environment --value)" + case " $environment " in + *" PGPASSWORD="*) + echo "Service environment still contains PGPASSWORD." >&2 + return 1 + ;; + esac + case " $environment " in + *" CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG "*) ;; + *) + echo "Service does not use the isolated CLOUDSDK_CONFIG." >&2 + return 1 + ;; + esac + runtime_dir_state="$(stat -c '%U:%G:%a' "$REMOTE_CLOUDSDK_CONFIG")" + if [ "$runtime_dir_state" != "teleo:teleo:700" ]; then + echo "Isolated CLOUDSDK_CONFIG has unexpected ownership or mode: $runtime_dir_state" >&2 + return 1 + fi +} + +assert_attached_service_account() { + local sdk_config="$1" metadata_email + sudo install -d -o teleo -g teleo -m 0700 "$sdk_config" + metadata_email="$( + sudo -n -u teleo env -u PGPASSWORD \ + HOME=/home/teleo \ + CLOUDSDK_CONFIG="$sdk_config" \ + curl --fail --silent --show-error \ + --header 'Metadata-Flavor: Google' \ + 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email' + )" + if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then + echo "Attached VM service account mismatch: $metadata_email" >&2 + return 1 + fi + + # An empty, phase-local Cloud SDK config forces this token request to use + # the VM's attached identity instead of an operator's cached credentials. + sudo -n -u teleo env -u PGPASSWORD \ + HOME=/home/teleo \ + CLOUDSDK_CONFIG="$sdk_config" \ + CLOUDSDK_CORE_PROJECT="$PROJECT" \ + gcloud --quiet auth print-access-token >/dev/null + echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email" +} + +run_scoped_status() { + local phase="$1" tool_path="$2" sdk_config="$3" + assert_attached_service_account "$sdk_config" + echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical" + sudo -n -u teleo env -u PGPASSWORD \ + HOME=/home/teleo \ + CLOUDSDK_CONFIG="$sdk_config" \ + CLOUDSDK_CORE_PROJECT="$PROJECT" \ + TELEO_GCP_PROJECT=teleo-501523 \ + TELEO_CLOUDSQL_HOST=10.61.0.3 \ + TELEO_CLOUDSQL_PORT=5432 \ + TELEO_CLOUDSQL_DB=teleo_canonical \ + TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical \ + TELEO_CLOUDSQL_USER=leoclean_kb_runtime \ + TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \ + TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \ + python3 "$tool_path" status --format json --redacted +} + +assert_deployed_files() { + local wrapper tool dropin + wrapper="$REMOTE_PROFILE_BIN/teleo-kb" + tool="$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py" + dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME" + test "$(sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA" + test "$(sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA" + test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA" + sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin" + sudo grep -E '^Environment=(CLOUDSDK_CONFIG|TELEO_(KB_MODE|GCP_PROJECT|CLOUDSQL_|CANONICAL_CLOUDSQL_DB))' "$dropin" +} +REMOTE +) + +printf -v verify_context \ + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \ + "$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \ + "$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA" + +verify_body=$(cat <<'REMOTE' +set -euo pipefail +verification_tmp="$(mktemp -d /tmp/teleo-gcp-runtime-verify.XXXXXX)" +cleanup_verification() { sudo rm -rf -- "$verification_tmp"; } +trap cleanup_verification EXIT +assert_deployed_files +assert_service_running +assert_runtime_environment +run_scoped_status verify "$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py" "$verification_tmp/gcloud" +systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager +REMOTE +) +verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}" + +if $DRY_RUN; then + echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE restart=$RESTART" + echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA dropin_sha256=$DROPIN_SHA" + exit 0 +fi + +if $VERIFY_ONLY; then + "${GCP_SSH[@]}" --command="$verify_command" + exit 0 +fi + +if ! $RESTART; then + echo "Refusing to install runtime files without --restart; use --dry-run or --verify-only for read-only work." >&2 + exit 64 +fi + +REMOTE_TMP="/tmp/teleo-gcp-leoclean-runtime-$(date -u +%Y%m%dT%H%M%SZ)-$$" +printf -v sync_context \ + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\nREMOTE_TMP=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \ + "$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \ + "$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA" "$REMOTE_TMP" \ + "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" "$SOURCE_REVISION" + +sync_body=$(cat <<'REMOTE' +set -euo pipefail +remote_tmp="$REMOTE_TMP" +profile_bin="$REMOTE_PROFILE_BIN" +dropin_dir="$REMOTE_DROPIN_DIR" +backup_dir="$remote_tmp/backup" +dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$" +mutation_started=false + +backup_path() { + local target="$1" name="$2" + if sudo test -e "$target"; then + sudo cp -a -- "$target" "$backup_dir/$name" + touch "$backup_dir/$name.present" + fi +} + +restore_path() { + local target="$1" name="$2" restore_rc=0 restore_tmp + if [ -f "$backup_dir/$name.present" ]; then + restore_tmp="${target}.livingip-rollback.$$" + sudo rm -f -- "$restore_tmp" || restore_rc=1 + if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then + sudo mv -f -- "$restore_tmp" "$target" || restore_rc=1 + else + restore_rc=1 + fi + sudo rm -f -- "$restore_tmp" || restore_rc=1 + else + sudo rm -f -- "$target" || restore_rc=1 + fi + return "$restore_rc" +} + +rollback_runtime() { + local rollback_rc=0 + echo "Deployment failed; restoring the complete previous runtime set." >&2 + sudo systemctl stop "$SERVICE" || rollback_rc=1 + restore_path "$profile_bin/teleo-kb" wrapper || rollback_rc=1 + restore_path "$profile_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 + restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 + restore_path "$profile_bin/.livingip-runtime-revision" revision || rollback_rc=1 + sudo systemctl daemon-reload || rollback_rc=1 + if [ "$rollback_rc" -eq 0 ]; then + sudo systemctl start "$SERVICE" || rollback_rc=1 + assert_service_running || rollback_rc=1 + fi + return "$rollback_rc" +} + +on_exit() { + local rc="$1" rollback_rc=0 + trap - EXIT INT TERM + if [ "$rc" -ne 0 ] && [ "$mutation_started" = true ]; then + set +e + rollback_runtime + rollback_rc=$? + set -e + if [ "$rollback_rc" -ne 0 ]; then + echo "CRITICAL: previous runtime restoration did not return the service to active/running." >&2 + rc=1 + fi + fi + sudo rm -f -- "$dropin_stage" || true + sudo rm -rf -- "$remote_tmp" || true + exit "$rc" +} + +trap 'on_exit $?' EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +sudo rm -rf -- "$remote_tmp" +mkdir -p "$backup_dir" +tar -C "$remote_tmp" -xf - + +assert_service_running +run_scoped_status pre "$remote_tmp/$TOOL_REL" "$remote_tmp/gcloud-pre" + +backup_path "$profile_bin/teleo-kb" wrapper +backup_path "$profile_bin/cloudsql_memory_tool.py" tool +backup_path "$dropin_dir/$DROPIN_NAME" dropin +backup_path "$profile_bin/.livingip-runtime-revision" revision + +mutation_started=true +sudo systemctl stop "$SERVICE" +sudo install -d -o teleo -g teleo -m 0755 "$profile_bin" +sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$WRAPPER_REL" "$profile_bin/teleo-kb" +sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$TOOL_REL" "$profile_bin/cloudsql_memory_tool.py" +sudo install -d -o root -g root -m 0755 "$dropin_dir" +sudo install -o root -g root -m 0644 "$remote_tmp/$DROPIN_REL" "$dropin_stage" +sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME" +printf '%s\n' "$SOURCE_REVISION" | sudo tee "$profile_bin/.livingip-runtime-revision" >/dev/null +sudo chown teleo:teleo "$profile_bin/.livingip-runtime-revision" +sudo chmod 0644 "$profile_bin/.livingip-runtime-revision" +sudo systemctl daemon-reload +sudo systemctl start "$SERVICE" + +assert_deployed_files +assert_service_running +assert_runtime_environment +run_scoped_status post "$profile_bin/cloudsql_memory_tool.py" "$remote_tmp/gcloud-post" +systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager +mutation_started=false +REMOTE +) +sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}" + +COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \ + "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command" diff --git a/docs/gcp-leoclean-runtime-reconciliation.md b/docs/gcp-leoclean-runtime-reconciliation.md new file mode 100644 index 0000000..754b15b --- /dev/null +++ b/docs/gcp-leoclean-runtime-reconciliation.md @@ -0,0 +1,157 @@ +# GCP Leo Runtime Reconciliation + +## Scope + +This runbook reconciles PR `#145` with the non-production GCP parallel Leo +service and removes its dependency on the PostgreSQL administrator credential. +It does not promote GCP, change the Telegram destination, copy newer VPS data, +or make `teleo_canonical` production-authoritative. + +Target surfaces: + +- project: `teleo-501523`; +- VM: `teleo-prod-1` in `europe-west6-a`; +- service: `leoclean-gcp-prod-parallel.service`; +- Cloud SQL database: `teleo_canonical` on private address `10.61.0.3`; +- runtime database role: `leoclean_kb_runtime`; +- staging-function owner: `leoclean_kb_stage_owner` (`NOLOGIN`); +- runtime secret: `gcp-teleo-pgvector-standby-leoclean-kb-runtime-password`. + +Observed before reconciliation on 2026-07-14: the service was +`active/running`, but `/usr/local/bin/teleo-kb` matched unmerged PR `#145`, the +effective systemd environment selected database user `postgres` and the +administrator password secret, and a live `teleo-kb status` call exceeded a +20-second bound. Secret Manager access and private PostgreSQL reachability both +passed independently, isolating the timeout to the old helper path rather than +IAM or networking. + +## Required End State + +1. The live wrapper and Cloud SQL helper hashes match a merged repository + commit. +2. `TELEO_KB_MODE=cloudsql`; missing tools or credentials fail closed. +3. Canonical zero-hit searches do not consult `teleo_restore` unless an + operator explicitly opts in. +4. Leo can read the named canonical tables and stage only a + `pending_review` proposal through `kb_stage.stage_leoclean_proposal(...)`. +5. Leo cannot directly insert, update, or delete `public.*` or + `kb_stage.kb_proposals`, forge the proposer identity, `SET ROLE` into a + broader principal, or execute reviewer/apply functions. +6. The VM runtime identity can access only the scoped password secret needed by + this path, not the PostgreSQL administrator password secret. +7. Deployment proves the attached VM service account from the metadata server, + uses an empty phase-local Cloud SDK configuration, and rolls the complete + prior runtime set back if installation or post-restart verification fails. + +## Safe Order Of Operations + +### 1. Read-only access and IAM audit + +Confirm the operator can read the project, use IAP/OS Login for the VM, inspect +Cloud SQL metadata, and inspect Secret Manager IAM. Identify the VM service +account and determine whether its secret access is project-wide or +secret-specific. Do not remove a project-wide binding until every legitimately +required runtime secret has an equivalent secret-level binding. + +### 2. Merge reviewed repository code + +Run CI on the completed PR `#145` repair before deployment. The deployment +source must be the resulting merged commit, not a working tree or unmerged +branch. + +### 3. Create the scoped secret and grant only the VM runtime identity + +Create the scoped secret without printing its value. Add one randomly generated +version, then grant `roles/secretmanager.secretAccessor` on that secret to the +VM runtime service account. Do not put `PGPASSWORD` in systemd, a repository, +an artifact, or a command transcript. + +### 4. Provision the scoped PostgreSQL role once + +Run `ops/gcp_leoclean_runtime_role.sql` as the Cloud SQL administrator from the +private VM path. Supply the new runtime password through +`TELEO_LEOCLEAN_DB_PASSWORD`; the SQL file does not contain it. The migration: + +- creates or rotates `leoclean_kb_runtime`; +- creates a dedicated `NOLOGIN` function owner with no role memberships; +- removes stale table, column, sequence, function, and role-membership grants; +- grants exact canonical reads; +- creates a locked security-definer staging function that hard-codes both + `pending_review` and canonical proposer `leo`; +- grants no direct table writes; +- removes all access to the legacy `teleo_restore` schema; +- aborts if either scoped role owns or can reach anything outside the explicit + allowlist. + +PostgreSQL grants `TEMP` to `PUBLIC` by default. The migration removes any +direct scoped `TEMP` grant and reports the remaining effective privilege, but +does not revoke `TEMP` from `PUBLIC`: PostgreSQL has no per-role deny, so that +would be a database-wide behavior change requiring a separate inventory of +`kb_apply`, reviewer, and operator use. The staging function remains protected +by a `pg_catalog, pg_temp` search path, schema-qualified relations, a fixed +`session_user`, and a tested temporary-object shadowing denial. + +The administrator password is used only for this bounded bootstrap. Retain no +password output. + +### 5. Preflight and deploy the merged runtime + +From the exact merged checkout, run: + +```bash +deploy/sync-gcp-leoclean-runtime.sh --dry-run +deploy/sync-gcp-leoclean-runtime.sh --restart +``` + +The deploy script runs a scoped, redacted database status preflight before it +installs files or restarts the service. It refuses dirty or unmerged runtime +files and refuses a live install without `--restart`. After preflight it stops +the parallel service, atomically replaces the existing Cloud SQL systemd +drop-in, installs the reviewed wrapper and helper, records the Git revision, +starts the service, and verifies all three file hashes. A failure after mutation +restores the prior wrapper, helper, drop-in, and revision as one set before the +old service is restarted. + +### 6. Verify positive and negative behavior + +Retain a redacted receipt containing: + +- merged Git commit and deployed file hashes; +- service `ActiveState`, `SubState`, `MainPID`, and `NRestarts`; +- metadata-server identity and an access token obtained with an empty Cloud SDK + configuration; +- `current_database()` and `current_user` showing + `teleo_canonical|leoclean_kb_runtime`; +- a real canonical status/search receipt; +- a transaction-rolled-back call to `stage_leoclean_proposal(...)` that returns + `pending_review`; +- denied direct insert into `kb_stage.kb_proposals`; +- denied canonical `public.*` write; +- denied reviewer/apply security-definer function; +- denied forged proposer identity and denied `SET ROLE` escalation; +- zero Telegram messages and zero committed canary rows. + +### 7. Remove administrator-secret access + +Only after the scoped service passes post-restart verification, remove the VM +runtime identity's access path to +`gcp-teleo-pgvector-standby-postgres-password`. Verify from the VM runtime +identity that the scoped secret is readable and the administrator secret is +denied. Never delete the administrator secret merely to enforce runtime least +privilege; backup/restore operators may still require it under a separate +identity. + +## Stop Conditions + +Stop without cutover if any of these are true: + +- the deployed revision is not merged; +- the scoped status preflight fails; +- the role has direct proposal-table insert or canonical write permission; +- any approval/apply function is executable by the runtime role; +- removing broad Secret Manager access would break another required secret; +- GCP canonical rows are still stale relative to the chosen authority. + +The last condition does not invalidate this runtime security repair. It means +GCP remains staging and data reconciliation stays a separate, explicitly +authorized slice. diff --git a/docs/reports/leo-working-state-20260709/gcp-db-first-live-deploy-restart-current.json b/docs/reports/leo-working-state-20260709/gcp-db-first-live-deploy-restart-current.json index 651db21..9ad235f 100644 --- a/docs/reports/leo-working-state-20260709/gcp-db-first-live-deploy-restart-current.json +++ b/docs/reports/leo-working-state-20260709/gcp-db-first-live-deploy-restart-current.json @@ -6,6 +6,7 @@ "database_first_merge_commit": "08371bf2a0461536c112e1235a7229b52ed270cc", "database_first_pull_request": 144, "fail_closed_wrapper_commit": "8451c51", + "fail_closed_wrapper_merge_state_at_capture": "unmerged PR #145; manually installed on the GCP VM", "repository": "living-ip/teleo-infrastructure" }, "deployment": { @@ -55,7 +56,7 @@ "integration_regression": { "detected_live": true, "symptom": "The old auto-mode wrapper used a full receipted status call as a 30-second health probe; the stronger read exceeded that bound and the wrapper fell through to a different local tool.", - "repair": "Supported GCP knowledge commands now route directly to Cloud SQL and fail closed on errors instead of changing databases.", + "repair": "The manually installed PR #145 wrapper routes supported GCP knowledge commands directly to Cloud SQL and propagates failures after selection. Missing prerequisites could still fall through before selection at capture time.", "regression_tests": 2, "full_repository_tests": { "passed": 1240, @@ -113,8 +114,10 @@ "post_restart_model_turn_run": false }, "claim_ceiling": { - "proven": "Merged database-first helper and skill deployment, fail-closed Cloud SQL routing, controlled GCP service restart, and post-restart live status/search receipts.", + "proven": "Merged database-first helper and skill deployment, manually installed unmerged wrapper routing repair, controlled GCP service restart, and post-restart live status/search receipts.", "not_proven": [ + "repository/runtime parity for the wrapper", + "least-privilege Cloud SQL runtime credential", "post-restart model reasoning turn", "Telegram-visible delivery", "canonical proposal apply", diff --git a/docs/reports/leo-working-state-20260709/gcp-db-first-working-leo-20260714.md b/docs/reports/leo-working-state-20260709/gcp-db-first-working-leo-20260714.md index b33ff7b..d906df5 100644 --- a/docs/reports/leo-working-state-20260709/gcp-db-first-working-leo-20260714.md +++ b/docs/reports/leo-working-state-20260709/gcp-db-first-working-leo-20260714.md @@ -94,10 +94,11 @@ remaining target databases. The disabled rollback database still has zero connections. The live GCP Leo service remained PID `148735`, active/running, with `NRestarts=0` throughout restore, model replay, and cleanup. -### 5. The merged GCP path survived deployment and restart +### 5. The merged database-first path and an unmerged wrapper repair survived restart PR `#144` merged the database-first helper and skill. Their exact reviewed -hashes were installed on `teleo-prod-1`, then +hashes were installed on `teleo-prod-1`. A separate wrapper repair from the +then-unmerged PR `#145` was also installed manually, then `leoclean-gcp-prod-parallel.service` was intentionally restarted. It returned active/running with a new PID (`304036`) and the deployed hashes remained in place. @@ -105,9 +106,12 @@ place. The first post-restart search caught one real integration regression: the old wrapper used a full `status` read as a 30-second health probe. Stronger receipts made that probe exceed its bound, after which `auto` mode could silently select -a different local tool. The wrapper now routes every supported GCP knowledge -command directly to Cloud SQL and fails closed on errors. Two regression tests -cover direct routing and no fallback. +a different local tool. The manually deployed wrapper routes every supported +GCP knowledge command directly to Cloud SQL and propagates errors after backend +selection. At the time of this receipt, missing host prerequisites could still +cause an earlier fallback, the runtime still used the administrator database +credential, and the live wrapper did not match merged `main`. PR `#145` +therefore remained a reconciliation task rather than a production-ready result. A corrected live search then found the expected sandbagging claim plus both expected source rows and hashes from persistent `teleo_canonical`. A subsequent @@ -140,7 +144,7 @@ rows as if they were source-code branches. | `gcp-db-first-parity-current.json` | `0173ee6707016e8412e6dd4326d61f71b6ef862bbc5b819079d62680676729f2` | Schema, row, role, extension, and performance parity passed | | `gcp-db-first-blind-claim-current.json` | `8a7cc3c1814eb385e858f12ef7579cd4524f37886c03fc6d9c61624b6aff2a52` | Real GCP Hermes reasoning passed with source-bound read receipts | | `gcp-db-first-cleanup-current.json` | `cac7e34f45653fabb696d911b6ccc921d3f291bf8cbe05a429d757e717dcafb4` | Clone absent, rollback disabled, service unchanged | -| `gcp-db-first-live-deploy-restart-current.json` | `d70fdbac859b8c98d557f4df900139a93e6009e12f7b80c9adbd69a5655b9df7` | Merged deploy, fail-closed routing, restart, and live status/search readback passed | +| `gcp-db-first-live-deploy-restart-current.json` | `78e9dde38d641bfa3d58b2b7d8605b37f2861605707eddc0c94ad5c9a70d080d` | Merged PR `#144` files plus an unmerged PR `#145` wrapper were installed; restart and live status/search readback passed | ## What This Unlocks @@ -166,6 +170,9 @@ rows as if they were source-code branches. 6. A model reasoning turn after the controlled GCP restart. The post-restart proof in this report is the real service plus live `teleo-kb` status/search path, not a new paid model response. +7. Repository/runtime parity for the wrapper or a least-privilege Cloud SQL + credential. The captured service still used an unmerged wrapper and the + administrator database secret. The next product-level proof is one natural Telegram challenge that reaches the same database-grounded reasoning, followed by one explicitly approved diff --git a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py index a7fb4fe..dd2edbe 100755 --- a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py +++ b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py @@ -74,6 +74,9 @@ STOPWORDS = { DEFAULT_CLAIM_BASE_URL = "https://leo.livingip.xyz" RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1" +RUNTIME_DB_USER = "leoclean_kb_runtime" +RUNTIME_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" +CLONE_READ_ONLY_PGOPTIONS = "-c default_transaction_read_only=on" RECEIPTED_READ_COMMANDS = frozenset( { "search", @@ -94,14 +97,20 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--host", default=os.environ.get("TELEO_CLOUDSQL_HOST", "10.61.0.3")) parser.add_argument("--port", default=os.environ.get("TELEO_CLOUDSQL_PORT", "5432")) - parser.add_argument("--db", default=os.environ.get("TELEO_CLOUDSQL_DB", "teleo_kb")) + parser.add_argument("--db", default=os.environ.get("TELEO_CLOUDSQL_DB", "teleo_canonical")) parser.add_argument("--canonical-db", default=os.environ.get("TELEO_CANONICAL_CLOUDSQL_DB", "teleo_canonical")) - parser.add_argument("--user", default=os.environ.get("TELEO_CLOUDSQL_USER", "postgres")) + parser.add_argument("--user", default=os.environ.get("TELEO_CLOUDSQL_USER")) parser.add_argument( "--password-secret", - default=os.environ.get("TELEO_CLOUDSQL_PASSWORD_SECRET", "gcp-teleo-pgvector-standby-postgres-password"), + default=os.environ.get("TELEO_CLOUDSQL_PASSWORD_SECRET"), ) parser.add_argument("--project", default=os.environ.get("TELEO_GCP_PROJECT", "teleo-501523")) + parser.add_argument( + "--credential-mode", + choices=["runtime", "clone-readonly"], + default=os.environ.get("TELEO_CLOUDSQL_CREDENTIAL_MODE", "runtime"), + help="Runtime mode requires the exact scoped Leo identity. Clone mode is limited to read-only teleo_clone_* tests.", + ) parser.add_argument("--format", choices=["markdown", "json"], default="markdown") parser.add_argument( "--include-excerpts", @@ -185,7 +194,7 @@ def parse_args() -> argparse.Namespace: propose.add_argument("--proposed", required=True, help="Proposed replacement or correction.") propose.add_argument("--evidence", action="append", default=[], help="Evidence/source pointer. Can be repeated.") propose.add_argument("--implication", action="append", default=[], help="Downstream implication. Can be repeated.") - propose.add_argument("--proposed-by", default="leo") + propose.add_argument("--proposed-by", choices=["leo"], default="leo") propose.add_argument("--originator", default="", help="Human or agent who originated the correction.") propose.add_argument("--channel", default="telegram") propose.add_argument("--source-ref", required=True) @@ -196,7 +205,7 @@ def parse_args() -> argparse.Namespace: propose_edge.add_argument("from_claim") propose_edge.add_argument("edge_type") propose_edge.add_argument("to_claim") - propose_edge.add_argument("--proposed-by", default="leo") + propose_edge.add_argument("--proposed-by", choices=["leo"], default="leo") propose_edge.add_argument("--channel", default="telegram") propose_edge.add_argument("--source-ref", required=True) propose_edge.add_argument("--rationale", required=True) @@ -228,6 +237,34 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() +def validate_runtime_connection(args: argparse.Namespace) -> None: + user = str(args.user or "") + password_secret = str(args.password_secret or "") + mode = str(getattr(args, "credential_mode", "runtime")) + if mode == "runtime": + if user != RUNTIME_DB_USER: + raise SystemExit(f"Leo runtime requires database user {RUNTIME_DB_USER}.") + if password_secret != RUNTIME_PASSWORD_SECRET: + raise SystemExit(f"Leo runtime requires scoped password secret {RUNTIME_PASSWORD_SECRET}.") + if "PGPASSWORD" in os.environ: + raise SystemExit("Refusing inherited PGPASSWORD in Leo runtime mode.") + return + + if mode != "clone-readonly": + raise SystemExit(f"Unsupported Cloud SQL credential mode: {mode}.") + canonical_db = str(args.canonical_db or "") + audit_db = str(args.db or "") + pgoptions = os.environ.get("PGOPTIONS", "") + if not re.fullmatch(r"teleo_clone_[a-z0-9_]+", canonical_db) or audit_db != canonical_db: + raise SystemExit("Clone credential mode is limited to one teleo_clone_* database.") + if args.command not in RECEIPTED_READ_COMMANDS: + raise SystemExit("Clone credential mode permits read commands only.") + if pgoptions != CLONE_READ_ONLY_PGOPTIONS: + raise SystemExit("Clone credential mode requires server-enforced default_transaction_read_only=on.") + if not user or (not password_secret and not os.environ.get("PGPASSWORD")): + raise SystemExit("Clone credential mode requires an explicit user and credential source.") + + def terms_for(query: str) -> list[str]: terms: list[str] = [] for token in re.findall(r"[A-Za-z0-9][A-Za-z0-9.+#/-]*", query.lower()): @@ -269,8 +306,11 @@ def sql_array(values: list[str], cast: str = "text") -> str: def password(args: argparse.Namespace) -> str: - if os.environ.get("PGPASSWORD"): + validate_runtime_connection(args) + if args.credential_mode == "clone-readonly" and os.environ.get("PGPASSWORD"): return os.environ["PGPASSWORD"] + if not args.password_secret: + raise SystemExit("Scoped Cloud SQL password secret is not configured.") result = subprocess.run( [ "gcloud", @@ -287,15 +327,32 @@ def password(args: argparse.Namespace) -> str: ) if result.returncode != 0: raise SystemExit(f"Secret access failed: {result.stderr.strip()}") - return result.stdout.strip() + credential = result.stdout.strip() + if not credential: + raise SystemExit("Scoped Cloud SQL password secret resolved to an empty value.") + return credential def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: env = os.environ.copy() - env["PGPASSWORD"] = password(args) - conn = f"host={args.host} port={args.port} dbname={db or args.db} user={args.user} sslmode=require" + for key in tuple(env): + if key.startswith("PG"): + env.pop(key, None) + env.update( + { + "PGHOST": str(args.host), + "PGPORT": str(args.port), + "PGDATABASE": str(db or args.db), + "PGUSER": str(args.user), + "PGSSLMODE": "require", + "PGCONNECT_TIMEOUT": "8", + "PGPASSWORD": password(args), + } + ) + if args.credential_mode == "clone-readonly": + env["PGOPTIONS"] = CLONE_READ_ONLY_PGOPTIONS result = subprocess.run( - ["psql", conn, "-At", "-q", "-v", "ON_ERROR_STOP=1"], + ["psql", "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"], text=True, capture_output=True, input=sql, @@ -868,6 +925,11 @@ def query_rows(args: argparse.Namespace, query: str, limit: int) -> dict[str, An canonical = query_canonical_rows(args, query, limit) if canonical.get("hit_count_total", 0) or canonical.get("hits"): return canonical + if os.environ.get("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "") != "1": + canonical["canonical_fallback_reason"] = ( + "no matching canonical public/persona/strategy/belief rows; audit fallback disabled" + ) + return canonical if not audit_restore_available(args): canonical["canonical_fallback_reason"] = ( "no matching canonical public/persona/strategy/belief rows; teleo_restore audit fallback unavailable" @@ -1054,57 +1116,37 @@ def propose_core_change(args: argparse.Namespace) -> dict[str, Any]: with params as ( select {sql_literal(args.proposal_type)}::text as proposal_type, - nullif({sql_literal(args.proposed_by)}, '') as proposed_by_handle, nullif({sql_literal(args.channel)}, '') as channel, nullif({sql_literal(args.source_ref)}, '') as source_ref, {sql_literal(args.rationale)} as rationale, {sql_literal(payload_json)}::jsonb as payload ), proposer as ( - select a.id, a.handle - from public.agents a, params p - where a.handle = p.proposed_by_handle - limit 1 -), inserted as ( - insert into kb_stage.kb_proposals ( - proposal_type, - status, - proposed_by_handle, - proposed_by_agent_id, - channel, - source_ref, - rationale, - payload - ) - select + select kb_stage.stage_leoclean_proposal( p.proposal_type, - 'pending_review', - p.proposed_by_handle, - proposer.id, - coalesce(p.channel, 'telegram'), + p.channel, p.source_ref, p.rationale, p.payload + ) as proposal from params p - left join proposer on true - returning * ) select jsonb_build_object( 'artifact', 'teleo_cloudsql_kb_core_change_proposal', 'backend', {sql_literal(canonical_backend(args))}, - 'id', id::text, - 'proposal_type', proposal_type, - 'status', status, - 'proposed_by_handle', proposed_by_handle, - 'proposed_by_agent_id', proposed_by_agent_id::text, - 'channel', channel, - 'source_ref', source_ref, - 'rationale', rationale, - 'payload', payload, - 'created_at', created_at::text, + 'id', proposal->>'id', + 'proposal_type', proposal->>'proposal_type', + 'status', proposal->>'status', + 'proposed_by_handle', proposal->>'proposed_by_handle', + 'proposed_by_agent_id', proposal->>'proposed_by_agent_id', + 'channel', proposal->>'channel', + 'source_ref', proposal->>'source_ref', + 'rationale', proposal->>'rationale', + 'payload', proposal->'payload', + 'created_at', proposal->>'created_at', 'canonical_apply_done', false, 'runtime_memory_write_done', false )::text -from inserted; +from proposer; """ rows = psql_json_lines(args, sql, db=args.canonical_db) if not rows: @@ -1119,7 +1161,6 @@ with params as ( {sql_literal(args.from_claim)}::uuid as from_claim, {sql_literal(args.to_claim)}::uuid as to_claim, {sql_literal(args.edge_type)}::edge_type as edge_type, - nullif({sql_literal(args.proposed_by)}, '') as proposed_by_handle, nullif({sql_literal(args.channel)}, '') as channel, nullif({sql_literal(args.source_ref)}, '') as source_ref, {sql_literal(args.rationale)} as rationale @@ -1131,11 +1172,6 @@ with params as ( select c.id, c.text from public.claims c, params p where c.id = p.to_claim -), proposer as ( - select a.id, a.handle - from public.agents a, params p - where a.handle = p.proposed_by_handle - limit 1 ), existing_edge as ( select e.id from public.claim_edges e, params p @@ -1143,23 +1179,10 @@ with params as ( and e.to_claim = p.to_claim and e.edge_type = p.edge_type limit 1 -), inserted as ( - insert into kb_stage.kb_proposals ( - proposal_type, - status, - proposed_by_handle, - proposed_by_agent_id, - channel, - source_ref, - rationale, - payload - ) - select +), staged as ( + select kb_stage.stage_leoclean_proposal( 'add_edge', - 'pending_review', - p.proposed_by_handle, - proposer.id, - coalesce(p.channel, 'telegram'), + p.channel, p.source_ref, p.rationale, jsonb_build_object( @@ -1175,29 +1198,28 @@ with params as ( 'edge_type', p.edge_type::text ) ) + ) as proposal from params p join from_row on true join to_row on true - left join proposer on true - returning * ) select jsonb_build_object( 'artifact', 'teleo_cloudsql_kb_edge_proposal', 'backend', {sql_literal(canonical_backend(args))}, - 'id', id::text, - 'proposal_type', proposal_type, - 'status', status, - 'proposed_by_handle', proposed_by_handle, - 'proposed_by_agent_id', proposed_by_agent_id::text, - 'channel', channel, - 'source_ref', source_ref, - 'rationale', rationale, - 'payload', payload, - 'created_at', created_at::text, + 'id', proposal->>'id', + 'proposal_type', proposal->>'proposal_type', + 'status', proposal->>'status', + 'proposed_by_handle', proposal->>'proposed_by_handle', + 'proposed_by_agent_id', proposal->>'proposed_by_agent_id', + 'channel', proposal->>'channel', + 'source_ref', proposal->>'source_ref', + 'rationale', proposal->>'rationale', + 'payload', proposal->'payload', + 'created_at', proposal->>'created_at', 'canonical_apply_done', false, 'runtime_memory_write_done', false )::text -from inserted; +from staged; """ rows = psql_json_lines(args, sql, db=args.canonical_db) if not rows: @@ -1365,6 +1387,8 @@ select jsonb_build_object( def audit_restore_available(args: argparse.Namespace) -> bool: + if os.environ.get("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "") != "1": + return False sql = """ select bool_and(to_regclass(table_name) is not null) from (values @@ -1675,6 +1699,7 @@ def emit_markdown(value: dict[str, Any]) -> None: def main() -> None: args = parse_args() + validate_runtime_connection(args) read_loaders = { "search": lambda: query_rows(args, args.query, args.limit), "context": lambda: query_rows(args, args.query, args.limit), diff --git a/hermes-agent/leoclean-bin/teleo-kb b/hermes-agent/leoclean-bin/teleo-kb index 808c77f..369f09b 100755 --- a/hermes-agent/leoclean-bin/teleo-kb +++ b/hermes-agent/leoclean-bin/teleo-kb @@ -23,14 +23,25 @@ cloudsql_args() { } run_cloudsql() { - if cloudsql_supported; then - extra_args=() - while IFS= read -r arg; do - [ -n "$arg" ] && extra_args+=("$arg") - done < <(cloudsql_args) - exec python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}" + if ! cloudsql_supported; then + echo "teleo-kb: command '$COMMAND' is not supported by the Cloud SQL backend" >&2 + exit 64 fi - exec python3 "$KB_TOOL" "$@" + if [ ! -x "$CLOUDSQL_TOOL" ]; then + echo "teleo-kb: Cloud SQL tool is missing or not executable: $CLOUDSQL_TOOL" >&2 + exit 69 + fi + for dependency in python3 psql gcloud; do + if ! command -v "$dependency" >/dev/null 2>&1; then + echo "teleo-kb: required Cloud SQL dependency is unavailable: $dependency" >&2 + exit 69 + fi + done + extra_args=() + while IFS= read -r arg; do + [ -n "$arg" ] && extra_args+=("$arg") + done < <(cloudsql_args) + exec python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}" } case "$MODE" in @@ -46,8 +57,8 @@ case "$MODE" in auto) # Canonical commands must fail closed on Cloud SQL errors. Falling through # to a different local database would make source-of-truth behavior depend - # on health-check latency. - if [ -x "$CLOUDSQL_TOOL" ] && command -v psql >/dev/null 2>&1 && command -v gcloud >/dev/null 2>&1 && cloudsql_supported; then + # on health-check latency or host package state. + if cloudsql_supported; then run_cloudsql "$@" fi if command -v docker >/dev/null 2>&1; then diff --git a/ops/gcp_leoclean_runtime_role.sql b/ops/gcp_leoclean_runtime_role.sql new file mode 100644 index 0000000..cd090a3 --- /dev/null +++ b/ops/gcp_leoclean_runtime_role.sql @@ -0,0 +1,847 @@ +\set ON_ERROR_STOP on +\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD + +-- One-time GCP staging provisioning for Leo's Cloud SQL identity. The password +-- is supplied only through the process environment and is never stored here. +-- +-- The login can read the canonical allowlist and can stage a pending proposal +-- through one narrowly-scoped SECURITY DEFINER function. It cannot inherit or +-- SET ROLE into another principal, forge the proposer identity, write directly +-- to canonical/staging tables, or inspect the legacy teleo_restore audit schema. +select 'create role leoclean_kb_runtime login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8' +where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_runtime') +\gexec + +select 'create role leoclean_kb_stage_owner nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1' +where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_stage_owner') +\gexec + +select format('alter role leoclean_kb_runtime password %L', :'runtime_password') +\gexec + +alter role leoclean_kb_runtime + with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; +alter role leoclean_kb_stage_owner + with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1 + password null; + +alter role leoclean_kb_runtime reset all; +alter role leoclean_kb_stage_owner reset all; + +-- NOINHERIT does not prevent SET ROLE. Remove every membership edge in either +-- direction so neither scoped principal can reach, or be reached through, a +-- broader role left behind by an earlier deployment. +select format('revoke %I from %I', granted_role.rolname, member_role.rolname) + from pg_catalog.pg_auth_members membership + join pg_catalog.pg_roles granted_role on granted_role.oid = membership.roleid + join pg_catalog.pg_roles member_role on member_role.oid = membership.member + where granted_role.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + or member_role.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') +\gexec + +revoke all privileges on database teleo_canonical from leoclean_kb_runtime; +revoke all privileges on database teleo_canonical from leoclean_kb_stage_owner; +revoke all privileges on database teleo_kb from leoclean_kb_runtime; +revoke all privileges on database teleo_kb from leoclean_kb_stage_owner; +grant connect on database teleo_canonical to leoclean_kb_runtime; + +-- PostgreSQL normally grants TEMP to PUBLIC. A role-specific REVOKE cannot +-- override that inherited grant, while REVOKE TEMP FROM PUBLIC would be a +-- database-wide change that could break kb_apply/review or operator workflows. +-- Leave PUBLIC TEMP policy unchanged here and surface it in the final receipt. +-- TEMP cannot shadow this SECURITY DEFINER boundary: pg_catalog is first, +-- pg_temp is explicitly last, and every relation reference is schema-qualified. +-- Database CREATE, direct TEMP grants, and all unexpected direct database ACLs +-- are rejected below. A database-wide TEMP removal requires a separate role-use +-- inventory and explicit grants to every workload that still needs it. + +alter role leoclean_kb_runtime in database teleo_canonical reset all; +alter role leoclean_kb_stage_owner in database teleo_canonical reset all; +alter role leoclean_kb_runtime in database teleo_kb reset all; +alter role leoclean_kb_stage_owner in database teleo_kb reset all; + +alter role leoclean_kb_runtime in database teleo_canonical + set search_path = pg_catalog, public, kb_stage; +alter role leoclean_kb_runtime in database teleo_canonical + set statement_timeout = '15s'; +alter role leoclean_kb_runtime in database teleo_canonical + set lock_timeout = '2s'; + +\connect teleo_canonical + +begin; + +-- The deploy principal needs temporary ownership authority to replace an +-- existing function on idempotent reruns. This membership is transaction-local +-- and is revoked before verification/commit. +select format('grant leoclean_kb_stage_owner to %I', current_user) + where current_user <> 'leoclean_kb_stage_owner' +\gexec + +revoke all on schema public, kb_stage + from leoclean_kb_runtime, leoclean_kb_stage_owner; +revoke all privileges on all tables in schema public, kb_stage + from leoclean_kb_runtime, leoclean_kb_stage_owner; +revoke all privileges on all sequences in schema public, kb_stage + from leoclean_kb_runtime, leoclean_kb_stage_owner; +revoke all privileges on all functions in schema public, kb_stage + from leoclean_kb_runtime, leoclean_kb_stage_owner; +revoke all privileges on all procedures in schema public, kb_stage + from leoclean_kb_runtime, leoclean_kb_stage_owner; + +-- Table-level REVOKE does not clear column ACLs. Clear every column ACL for both +-- principals before rebuilding the exact allowlist below. +select format( + 'revoke all privileges (%s) on table %I.%I from %I', + pg_catalog.string_agg(pg_catalog.quote_ident(attribute.attname), ', ' order by attribute.attnum), + namespace.nspname, + relation.relname, + target.rolname + ) + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join ( + values ('leoclean_kb_runtime'), ('leoclean_kb_stage_owner') + ) as target(rolname) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + group by namespace.nspname, relation.relname, target.rolname +\gexec + +grant usage on schema public, kb_stage to leoclean_kb_runtime; + +grant select on + public.agents, + public.claims, + public.claim_evidence, + public.claim_edges, + public.sources, + public.personas, + public.strategies, + public.beliefs, + public.blindspots, + public.agent_roles, + public.peer_models, + public.behavioral_rules, + public.contributor_rules, + public.reasoning_tools, + public.governance_gates, + kb_stage.kb_proposals +to leoclean_kb_runtime; + +-- The function owner is a dedicated NOLOGIN role. Grant only the columns the +-- function reads/inserts. CREATE is temporary and is revoked immediately after +-- ownership is assigned. +grant usage, create on schema kb_stage to leoclean_kb_stage_owner; +grant usage on schema public to leoclean_kb_stage_owner; +grant select (id, handle) on public.agents to leoclean_kb_stage_owner; +grant select on kb_stage.kb_proposals to leoclean_kb_stage_owner; +grant insert ( + proposal_type, + status, + proposed_by_handle, + proposed_by_agent_id, + channel, + source_ref, + rationale, + payload +) on kb_stage.kb_proposals to leoclean_kb_stage_owner; + +-- Remove the prior caller-supplied-identity overload before installing the +-- session-bound form. A runtime caller can only stage as canonical agent Leo. +drop function if exists kb_stage.stage_leoclean_proposal(text, text, text, text, text, jsonb); + +create or replace function kb_stage.stage_leoclean_proposal( + p_proposal_type text, + p_channel text, + p_source_ref text, + p_rationale text, + p_payload jsonb +) returns jsonb +language plpgsql +security definer +set search_path = pg_catalog, pg_temp +as $function$ +declare + staged kb_stage.kb_proposals%rowtype; + proposer_id uuid; +begin + if session_user <> 'leoclean_kb_runtime' then + raise exception 'stage_leoclean_proposal is restricted to leoclean_kb_runtime' + using errcode = '42501'; + end if; + if p_proposal_type not in ('revise_claim', 'revise_strategy', 'add_edge') then + raise exception 'unsupported Leo proposal type: %', p_proposal_type using errcode = '22023'; + end if; + if nullif(btrim(p_source_ref), '') is null then + raise exception 'source_ref is required' using errcode = '22023'; + end if; + if nullif(btrim(p_rationale), '') is null then + raise exception 'rationale is required' using errcode = '22023'; + end if; + if p_payload is null or jsonb_typeof(p_payload) <> 'object' then + raise exception 'proposal payload must be a JSON object' using errcode = '22023'; + end if; + + select agent.id + into proposer_id + from public.agents as agent + where agent.handle = 'leo'; + + if proposer_id is null then + raise exception 'canonical Leo agent row is required' using errcode = '23503'; + end if; + + insert into kb_stage.kb_proposals ( + proposal_type, + status, + proposed_by_handle, + proposed_by_agent_id, + channel, + source_ref, + rationale, + payload + ) values ( + p_proposal_type, + 'pending_review', + 'leo', + proposer_id, + coalesce(nullif(p_channel, ''), 'telegram'), + p_source_ref, + p_rationale, + p_payload + ) + returning * into staged; + + return to_jsonb(staged); +end +$function$; + +alter function kb_stage.stage_leoclean_proposal(text, text, text, text, jsonb) + owner to leoclean_kb_stage_owner; +revoke create on schema kb_stage from leoclean_kb_stage_owner; + +-- Normalize explicit ACL entries left by CREATE OR REPLACE or an earlier grant. +revoke all on function kb_stage.stage_leoclean_proposal(text, text, text, text, jsonb) + from public; +select format( + 'revoke all on function kb_stage.stage_leoclean_proposal(text, text, text, text, jsonb) from %I', + grantee.rolname + ) + from pg_catalog.pg_proc function_row + cross join lateral pg_catalog.aclexplode( + coalesce( + function_row.proacl, + pg_catalog.acldefault('f', function_row.proowner) + ) + ) as acl + join pg_catalog.pg_roles grantee on grantee.oid = acl.grantee + where function_row.oid = 'kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)'::pg_catalog.regprocedure + and grantee.rolname <> 'leoclean_kb_stage_owner' +\gexec + +grant execute on function kb_stage.stage_leoclean_proposal(text, text, text, text, jsonb) + to leoclean_kb_runtime; + +select format('revoke leoclean_kb_stage_owner from %I', current_user) + where current_user <> 'leoclean_kb_stage_owner' +\gexec + +do $verification$ +declare + unexpected text; + runtime_oid oid := 'leoclean_kb_runtime'::pg_catalog.regrole::oid; + owner_oid oid := 'leoclean_kb_stage_owner'::pg_catalog.regrole::oid; + stage_function oid := 'kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)'::pg_catalog.regprocedure::oid; +begin + if exists ( + select 1 + from pg_catalog.pg_roles role_row + where role_row.rolname = 'leoclean_kb_runtime' + and (not role_row.rolcanlogin + or role_row.rolsuper + or role_row.rolcreatedb + or role_row.rolcreaterole + or role_row.rolinherit + or role_row.rolreplication + or role_row.rolbypassrls + or role_row.rolconnlimit <> 8) + ) then + raise exception 'leoclean_kb_runtime has unsafe role attributes'; + end if; + + if exists ( + select 1 + from pg_catalog.pg_roles role_row + where role_row.rolname = 'leoclean_kb_stage_owner' + and (role_row.rolcanlogin + or role_row.rolsuper + or role_row.rolcreatedb + or role_row.rolcreaterole + or role_row.rolinherit + or role_row.rolreplication + or role_row.rolbypassrls + or role_row.rolconnlimit <> -1) + ) then + raise exception 'leoclean_kb_stage_owner has unsafe role attributes'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I->%I', granted_role.rolname, member_role.rolname), + ', ' + order by granted_role.rolname, member_role.rolname + ) + into unexpected + from pg_catalog.pg_auth_members membership + join pg_catalog.pg_roles granted_role on granted_role.oid = membership.roleid + join pg_catalog.pg_roles member_role on member_role.oid = membership.member + where membership.roleid in (runtime_oid, owner_oid) + or membership.member in (runtime_oid, owner_oid); + if unexpected is not null then + raise exception 'scoped Leo roles retain role memberships: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format( + '%I:%s:%s:grantable=%s', + database_row.datname, + grantee.rolname, + acl.privilege_type, + acl.is_grantable + ), + ', ' + order by database_row.datname, grantee.rolname, acl.privilege_type + ) + into unexpected + from pg_catalog.pg_database database_row + cross join lateral pg_catalog.aclexplode( + coalesce( + database_row.datacl, + pg_catalog.acldefault('d', database_row.datdba) + ) + ) as acl + join pg_catalog.pg_roles grantee on grantee.oid = acl.grantee + where database_row.datname in ('teleo_canonical', 'teleo_kb') + and grantee.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and not ( + database_row.datname = 'teleo_canonical' + and grantee.rolname = 'leoclean_kb_runtime' + and acl.privilege_type = 'CONNECT' + and not acl.is_grantable + ); + if unexpected is not null then + raise exception 'scoped Leo roles retain unexpected direct database ACLs: %', unexpected; + end if; + + if has_database_privilege('leoclean_kb_runtime', 'teleo_canonical', 'CREATE') + or has_database_privilege('leoclean_kb_runtime', 'teleo_kb', 'CREATE') + or has_database_privilege('leoclean_kb_stage_owner', 'teleo_canonical', 'CREATE') + or has_database_privilege('leoclean_kb_stage_owner', 'teleo_kb', 'CREATE') then + raise exception 'a scoped Leo role unexpectedly has database CREATE'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s:%s:%s', dependency.classid::pg_catalog.regclass, dependency.objid, dependency.objsubid), + ', ' + ) + into unexpected + from pg_catalog.pg_shdepend dependency + where dependency.refclassid = 'pg_catalog.pg_authid'::pg_catalog.regclass + and dependency.refobjid = runtime_oid + and dependency.deptype = 'o'; + if unexpected is not null then + raise exception 'leoclean_kb_runtime unexpectedly owns database objects: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s:%s:%s', dependency.classid::pg_catalog.regclass, dependency.objid, dependency.objsubid), + ', ' + ) + into unexpected + from pg_catalog.pg_shdepend dependency + where dependency.refclassid = 'pg_catalog.pg_authid'::pg_catalog.regclass + and dependency.refobjid = owner_oid + and dependency.deptype = 'o' + and not ( + dependency.dbid = (select database_row.oid from pg_catalog.pg_database database_row where database_row.datname = current_database()) + and + dependency.classid = 'pg_catalog.pg_proc'::pg_catalog.regclass + and dependency.objid = stage_function + and dependency.objsubid = 0 + ); + if unexpected is not null then + raise exception 'leoclean_kb_stage_owner owns unexpected database objects: %', unexpected; + end if; + + if has_schema_privilege('leoclean_kb_runtime', 'public', 'CREATE') + or has_schema_privilege('leoclean_kb_runtime', 'kb_stage', 'CREATE') + or has_schema_privilege('leoclean_kb_stage_owner', 'public', 'CREATE') + or has_schema_privilege('leoclean_kb_stage_owner', 'kb_stage', 'CREATE') then + raise exception 'a scoped Leo role unexpectedly has schema CREATE'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name), + ', ' + order by namespace.nspname, relation.relname, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and has_table_privilege('leoclean_kb_runtime', relation.oid, privilege.name); + if unexpected is not null then + raise exception 'leoclean_kb_runtime unexpectedly has table-level DML: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I:%s', namespace.nspname, relation.relname, attribute.attname, privilege.name), + ', ' + order by namespace.nspname, relation.relname, attribute.attnum, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege('leoclean_kb_runtime', relation.oid, attribute.attnum, privilege.name); + if unexpected is not null then + raise exception 'leoclean_kb_runtime unexpectedly has column-level DML: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I', namespace.nspname, relation.relname), + ', ' + order by namespace.nspname, relation.relname + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and has_table_privilege('leoclean_kb_runtime', relation.oid, 'SELECT') + and (namespace.nspname, relation.relname) not in ( + ('public', 'agents'), + ('public', 'claims'), + ('public', 'claim_evidence'), + ('public', 'claim_edges'), + ('public', 'sources'), + ('public', 'personas'), + ('public', 'strategies'), + ('public', 'beliefs'), + ('public', 'blindspots'), + ('public', 'agent_roles'), + ('public', 'peer_models'), + ('public', 'behavioral_rules'), + ('public', 'contributor_rules'), + ('public', 'reasoning_tools'), + ('public', 'governance_gates'), + ('kb_stage', 'kb_proposals') + ); + if unexpected is not null then + raise exception 'leoclean_kb_runtime can SELECT outside its table allowlist: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I', namespace.nspname, relation.relname, attribute.attname), + ', ' + order by namespace.nspname, relation.relname, attribute.attnum + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege('leoclean_kb_runtime', relation.oid, attribute.attnum, 'SELECT') + and (namespace.nspname, relation.relname) not in ( + ('public', 'agents'), + ('public', 'claims'), + ('public', 'claim_evidence'), + ('public', 'claim_edges'), + ('public', 'sources'), + ('public', 'personas'), + ('public', 'strategies'), + ('public', 'beliefs'), + ('public', 'blindspots'), + ('public', 'agent_roles'), + ('public', 'peer_models'), + ('public', 'behavioral_rules'), + ('public', 'contributor_rules'), + ('public', 'reasoning_tools'), + ('public', 'governance_gates'), + ('kb_stage', 'kb_proposals') + ); + if unexpected is not null then + raise exception 'leoclean_kb_runtime has column SELECT outside its allowlist: %', unexpected; + end if; + + if has_table_privilege('leoclean_kb_stage_owner', 'public.agents', 'SELECT') + or not has_column_privilege('leoclean_kb_stage_owner', 'public.agents', 'id', 'SELECT') + or not has_column_privilege('leoclean_kb_stage_owner', 'public.agents', 'handle', 'SELECT') + or not has_table_privilege('leoclean_kb_stage_owner', 'kb_stage.kb_proposals', 'SELECT') + or has_table_privilege('leoclean_kb_stage_owner', 'kb_stage.kb_proposals', 'INSERT') then + raise exception 'leoclean_kb_stage_owner does not have the exact read/insert grant shape'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I', namespace.nspname, relation.relname, attribute.attname), + ', ' + order by namespace.nspname, relation.relname, attribute.attnum + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege('leoclean_kb_stage_owner', relation.oid, attribute.attnum, 'INSERT') + and not ( + namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' + and attribute.attname in ( + 'proposal_type', + 'status', + 'proposed_by_handle', + 'proposed_by_agent_id', + 'channel', + 'source_ref', + 'rationale', + 'payload' + ) + ); + if unexpected is not null then + raise exception 'leoclean_kb_stage_owner can INSERT unexpected columns: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name), + ', ' + order by namespace.nspname, relation.relname, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and has_table_privilege('leoclean_kb_stage_owner', relation.oid, privilege.name); + if unexpected is not null then + raise exception 'leoclean_kb_stage_owner unexpectedly has table-level DML: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I:%s', namespace.nspname, relation.relname, attribute.attname, privilege.name), + ', ' + order by namespace.nspname, relation.relname, attribute.attnum, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('UPDATE'), ('REFERENCES')) privilege(name) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege('leoclean_kb_stage_owner', relation.oid, attribute.attnum, privilege.name); + if unexpected is not null then + raise exception 'leoclean_kb_stage_owner unexpectedly has column-level DML: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I', namespace.nspname, relation.relname, attribute.attname), + ', ' + order by namespace.nspname, relation.relname, attribute.attnum + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege('leoclean_kb_stage_owner', relation.oid, attribute.attnum, 'SELECT') + and not ( + (namespace.nspname = 'kb_stage' and relation.relname = 'kb_proposals') + or ( + namespace.nspname = 'public' + and relation.relname = 'agents' + and attribute.attname in ('id', 'handle') + ) + ); + if unexpected is not null then + raise exception 'leoclean_kb_stage_owner can SELECT unexpected columns: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s:%s', namespace.nspname, relation.relname, scoped_role.name, privilege.name), + ', ' + order by namespace.nspname, relation.relname, scoped_role.name, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + cross join (values ('leoclean_kb_runtime'), ('leoclean_kb_stage_owner')) scoped_role(name) + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind = 'S' + and has_sequence_privilege(scoped_role.name, relation.oid, privilege.name); + if unexpected is not null then + raise exception 'a scoped Leo role unexpectedly has sequence privileges: %', unexpected; + end if; + + if exists ( + select 1 + from pg_catalog.pg_attribute attribute + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attname in ( + 'proposal_type', + 'status', + 'proposed_by_handle', + 'proposed_by_agent_id', + 'channel', + 'source_ref', + 'rationale', + 'payload' + ) + and not has_column_privilege( + 'leoclean_kb_stage_owner', + attribute.attrelid, + attribute.attnum, + 'INSERT' + ) + ) then + raise exception 'leoclean_kb_stage_owner is missing a required INSERT column grant'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I(%s)', namespace.nspname, function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid)), + ', ' + ) + into unexpected + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname in ('public', 'kb_stage') + and function_row.prosecdef + and function_row.oid <> stage_function + and has_function_privilege('leoclean_kb_runtime', function_row.oid, 'EXECUTE'); + if unexpected is not null then + raise exception 'leoclean_kb_runtime can execute unexpected SECURITY DEFINER functions: %', unexpected; + end if; + + if not exists ( + select 1 + from pg_catalog.pg_proc function_row + where function_row.oid = stage_function + and function_row.prosecdef + and function_row.proowner = owner_oid + and function_row.proconfig = array['search_path=pg_catalog, pg_temp']::text[] + ) then + raise exception 'stage_leoclean_proposal owner/security/search_path contract is invalid'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format( + 'grantee=%s privilege=%s grantable=%s', + acl.grantee, + acl.privilege_type, + acl.is_grantable + ), + ', ' + ) + into unexpected + from pg_catalog.pg_proc function_row + cross join lateral pg_catalog.aclexplode( + coalesce( + function_row.proacl, + pg_catalog.acldefault('f', function_row.proowner) + ) + ) as acl + where function_row.oid = stage_function + and not ( + acl.grantee in (runtime_oid, owner_oid) + and acl.privilege_type = 'EXECUTE' + and (acl.grantee = owner_oid or not acl.is_grantable) + ); + if unexpected is not null then + raise exception 'stage_leoclean_proposal has unexpected ACL entries: %', unexpected; + end if; + + if not has_function_privilege( + 'leoclean_kb_runtime', + 'kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)', + 'EXECUTE' + ) then + raise exception 'leoclean_kb_runtime cannot execute stage_leoclean_proposal'; + end if; +end +$verification$; + +commit; + +-- The legacy restore database is not an audit fallback for the scoped runtime. +-- Strip direct ACLs from both scoped roles and fail if PUBLIC/group grants still +-- make any teleo_restore relation reachable. +\connect teleo_kb + +begin; + +select format('revoke all on schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) + from pg_catalog.pg_namespace namespace + where namespace.nspname = 'teleo_restore' +\gexec + +select format('revoke all privileges on all tables in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) + from pg_catalog.pg_namespace namespace + where namespace.nspname = 'teleo_restore' +\gexec + +select format('revoke all privileges on all sequences in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) + from pg_catalog.pg_namespace namespace + where namespace.nspname = 'teleo_restore' +\gexec + +select format('revoke all privileges on all functions in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) + from pg_catalog.pg_namespace namespace + where namespace.nspname = 'teleo_restore' +\gexec + +select format('revoke all privileges on all procedures in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) + from pg_catalog.pg_namespace namespace + where namespace.nspname = 'teleo_restore' +\gexec + +select format( + 'revoke all privileges (%s) on table %I.%I from %I', + pg_catalog.string_agg(pg_catalog.quote_ident(attribute.attname), ', ' order by attribute.attnum), + namespace.nspname, + relation.relname, + target.rolname + ) + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join ( + values ('leoclean_kb_runtime'), ('leoclean_kb_stage_owner') + ) as target(rolname) + where namespace.nspname = 'teleo_restore' + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + group by namespace.nspname, relation.relname, target.rolname +\gexec + +do $audit_verification$ +declare + unexpected text; + scoped_role text; +begin + foreach scoped_role in array array['leoclean_kb_runtime', 'leoclean_kb_stage_owner'] loop + if to_regnamespace('teleo_restore') is not null + and (has_schema_privilege(scoped_role, 'teleo_restore', 'USAGE') + or has_schema_privilege(scoped_role, 'teleo_restore', 'CREATE')) then + raise exception '% unexpectedly has teleo_restore schema access', scoped_role; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name), + ', ' + order by relation.relname, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + cross join ( + values ('SELECT'), ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER') + ) privilege(name) + where namespace.nspname = 'teleo_restore' + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and has_table_privilege(scoped_role, relation.oid, privilege.name); + if unexpected is not null then + raise exception '% unexpectedly has teleo_restore table access: %', scoped_role, unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I:%s', namespace.nspname, relation.relname, attribute.attname, privilege.name), + ', ' + order by relation.relname, attribute.attnum, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('SELECT'), ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where namespace.nspname = 'teleo_restore' + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege(scoped_role, relation.oid, attribute.attnum, privilege.name); + if unexpected is not null then + raise exception '% unexpectedly has teleo_restore column access: %', scoped_role, unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name), + ', ' + order by relation.relname, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where namespace.nspname = 'teleo_restore' + and relation.relkind = 'S' + and has_sequence_privilege(scoped_role, relation.oid, privilege.name); + if unexpected is not null then + raise exception '% unexpectedly has teleo_restore sequence access: %', scoped_role, unexpected; + end if; + end loop; +end +$audit_verification$; + +commit; + +\connect teleo_canonical + +select jsonb_build_object( + 'role', rolname, + 'superuser', rolsuper, + 'create_db', rolcreatedb, + 'create_role', rolcreaterole, + 'inherit', rolinherit, + 'replication', rolreplication, + 'bypass_rls', rolbypassrls, + 'has_role_memberships', exists ( + select 1 + from pg_catalog.pg_auth_members membership + where membership.roleid = pg_catalog.pg_roles.oid + or membership.member = pg_catalog.pg_roles.oid + ), + 'can_select_claims', has_table_privilege(rolname, 'public.claims', 'SELECT'), + 'can_select_proposals', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'SELECT'), + 'can_insert_proposals_directly', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'INSERT'), + 'can_create_database_objects', has_database_privilege(rolname, 'teleo_canonical', 'CREATE'), + 'can_create_temp_objects', has_database_privilege(rolname, 'teleo_canonical', 'TEMP'), + 'can_stage_proposal', has_function_privilege( + rolname, + 'kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)', + 'EXECUTE' + ) +)::text +from pg_catalog.pg_roles +where rolname = 'leoclean_kb_runtime'; diff --git a/scripts/run_gcp_generated_db_direct_claim_suite.py b/scripts/run_gcp_generated_db_direct_claim_suite.py index 79b0704..f2a3e31 100644 --- a/scripts/run_gcp_generated_db_direct_claim_suite.py +++ b/scripts/run_gcp_generated_db_direct_claim_suite.py @@ -47,7 +47,7 @@ DEFAULT_HOST = "10.61.0.3" DEFAULT_PROJECT = "teleo-501523" DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql" -REVIEWED_CLOUDSQL_TOOL_SHA256 = "7d2074a0fc5f0d48fbf8f7905a72ead8f2696c86a041fa43e078fff5500baa51" +REVIEWED_CLOUDSQL_TOOL_SHA256 = "2617bb501456758131ff41a6c77b3c462e21c28c4348da319b60defb051c7c87" REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a" COUNT_READBACK_RE = re.compile( r"DB readback:\s*claims:\s*`?(?P\d+)`?;\s*" @@ -416,6 +416,8 @@ $PYTHON "$CLOUDSQL_TOOL" \ --db "$TARGET_DATABASE" \ --canonical-db "$TARGET_DATABASE" \ --project "$PROJECT" \ + --credential-mode clone-readonly \ + --user postgres \ --password-secret "$PASSWORD_SECRET" \ "$@" STATUS=$? diff --git a/systemd/leoclean-gcp-prod-parallel-cloudsql.conf b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf new file mode 100644 index 0000000..1b267f5 --- /dev/null +++ b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf @@ -0,0 +1,14 @@ +[Service] +UnsetEnvironment=PGPASSWORD +RuntimeDirectory=leoclean-gcp-prod-parallel +RuntimeDirectoryMode=0700 +Environment=CLOUDSDK_CONFIG=/run/leoclean-gcp-prod-parallel +Environment=TELEO_KB_MODE=cloudsql +Environment=TELEO_GCP_PROJECT=teleo-501523 +Environment=TELEO_CLOUDSQL_HOST=10.61.0.3 +Environment=TELEO_CLOUDSQL_PORT=5432 +Environment=TELEO_CLOUDSQL_DB=teleo_canonical +Environment=TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical +Environment=TELEO_CLOUDSQL_USER=leoclean_kb_runtime +Environment=TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password +Environment=TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 diff --git a/tests/test_gcp_generated_db_direct_claim_suite.py b/tests/test_gcp_generated_db_direct_claim_suite.py index 18ce5d0..475b8e7 100644 --- a/tests/test_gcp_generated_db_direct_claim_suite.py +++ b/tests/test_gcp_generated_db_direct_claim_suite.py @@ -115,6 +115,8 @@ def test_wrapper_binds_private_read_only_database_and_records_calls(tmp_path: Pa assert "dbname=$TARGET_DATABASE" in wrapper assert '--db "$TARGET_DATABASE"' in wrapper assert '--canonical-db "$TARGET_DATABASE"' in wrapper + assert "--credential-mode clone-readonly" in wrapper + assert "--user postgres" in wrapper assert "sslmode=require" in wrapper assert "begin transaction read only" in wrapper assert 'export PGOPTIONS="-c default_transaction_read_only=on"' in wrapper @@ -306,6 +308,8 @@ def test_generated_wrapper_executes_only_clone_bound_default_read_only_tool(tmp_ "--canonical-db", "teleo_clone_test", ] + assert tool_receipt["argv"][tool_receipt["argv"].index("--credential-mode") + 1] == "clone-readonly" + assert tool_receipt["argv"][tool_receipt["argv"].index("--user") + 1] == "postgres" events = [json.loads(line) for line in log.read_text().splitlines()] assert [event["phase"] for event in events] == ["start", "end"] assert events[0]["database_identity"]["default_transaction_read_only"] == "on" diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index ab5a8d4..5b19a27 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -9,6 +9,7 @@ import io import json import os import re +import shutil import subprocess from pathlib import Path from types import SimpleNamespace @@ -23,6 +24,7 @@ from scripts.working_leo_open_ended_benchmark import ( ROOT = Path(__file__).resolve().parents[1] BRIDGE_DIR = ROOT / "hermes-agent" / "leoclean-bin" DB_CONTEXT_PLUGIN = ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py" +GCP_RUNTIME_ROLE_SQL = ROOT / "ops" / "gcp_leoclean_runtime_role.sql" def test_leoclean_bridge_files_are_present_and_parseable() -> None: @@ -67,7 +69,7 @@ def test_cloudsql_wrapper_supports_expected_review_gated_commands() -> None: assert "canonical apply" in cloudsql_text.lower() -def _install_wrapper_fixture(tmp_path: Path) -> tuple[Path, dict[str, str]]: +def _install_wrapper_fixture(tmp_path: Path, *, missing: frozenset[str] = frozenset()) -> tuple[Path, dict[str, str]]: profile = tmp_path / "profile" bin_dir = profile / "bin" bin_dir.mkdir(parents=True) @@ -75,11 +77,12 @@ def _install_wrapper_fixture(tmp_path: Path) -> tuple[Path, dict[str, str]]: wrapper.write_bytes((BRIDGE_DIR / "teleo-kb").read_bytes()) wrapper.chmod(0o700) cloudsql = bin_dir / "cloudsql_memory_tool.py" - cloudsql.write_text( - "import json, sys\nprint(json.dumps({'backend': 'cloudsql', 'argv': sys.argv[1:]}))\n", - encoding="utf-8", - ) - cloudsql.chmod(0o700) + if "cloudsql-tool" not in missing: + cloudsql.write_text( + "import json, sys\nprint(json.dumps({'backend': 'cloudsql', 'argv': sys.argv[1:]}))\n", + encoding="utf-8", + ) + cloudsql.chmod(0o700) local = bin_dir / "kb_tool.py" local.write_text( "import json, sys\nprint(json.dumps({'backend': 'local', 'argv': sys.argv[1:]}))\n", @@ -88,7 +91,13 @@ def _install_wrapper_fixture(tmp_path: Path) -> tuple[Path, dict[str, str]]: local.chmod(0o700) fake_bin = tmp_path / "fake-bin" fake_bin.mkdir() + for name in ("bash", "python3"): + target = shutil.which(name) + assert target + (fake_bin / name).symlink_to(target) for name in ("psql", "gcloud"): + if name in missing: + continue executable = fake_bin / name executable.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") executable.chmod(0o700) @@ -96,7 +105,7 @@ def _install_wrapper_fixture(tmp_path: Path) -> tuple[Path, dict[str, str]]: **os.environ, "HERMES_HOME": str(profile), "TELEO_KB_MODE": "auto", - "PATH": f"{fake_bin}:{os.environ['PATH']}", + "PATH": str(fake_bin), } return wrapper, env @@ -137,6 +146,40 @@ def test_cloudsql_wrapper_auto_mode_never_falls_back_after_supported_command_fai assert "local" not in completed.stdout +@pytest.mark.parametrize("missing", ["cloudsql-tool", "psql", "gcloud"]) +def test_cloudsql_wrapper_auto_mode_fails_closed_when_prerequisite_is_missing(tmp_path: Path, missing: str) -> None: + wrapper, env = _install_wrapper_fixture(tmp_path, missing=frozenset({missing})) + + completed = subprocess.run( + [str(wrapper), "search", "canonical only", "--format", "json"], + text=True, + capture_output=True, + check=False, + env=env, + ) + + assert completed.returncode == 69 + assert "local" not in completed.stdout + assert "Cloud SQL" in completed.stderr + + +def test_cloudsql_wrapper_explicit_mode_rejects_unsupported_commands_without_fallback(tmp_path: Path) -> None: + wrapper, env = _install_wrapper_fixture(tmp_path) + env["TELEO_KB_MODE"] = "cloudsql" + + completed = subprocess.run( + [str(wrapper), "prepare-source", "example.md"], + text=True, + capture_output=True, + check=False, + env=env, + ) + + assert completed.returncode == 64 + assert "local" not in completed.stdout + assert "not supported by the Cloud SQL backend" in completed.stderr + + def test_bridge_edge_proposals_stage_strict_apply_payloads() -> None: cloudsql_text = (BRIDGE_DIR / "cloudsql_memory_tool.py").read_text() vps_text = (BRIDGE_DIR / "kb_tool.py").read_text() @@ -148,6 +191,9 @@ def test_bridge_edge_proposals_stage_strict_apply_payloads() -> None: assert "'to_claim', to_row.id::text" in text assert "'edge_type', p.edge_type::text" in text + assert "kb_stage.stage_leoclean_proposal(" in cloudsql_text + assert "insert into kb_stage.kb_proposals" not in cloudsql_text.lower() + def test_bridge_source_does_not_commit_raw_secret_values() -> None: combined = "\n".join(path.read_text(errors="replace") for path in BRIDGE_DIR.iterdir() if path.is_file()) @@ -163,6 +209,206 @@ def test_bridge_source_does_not_commit_raw_secret_values() -> None: assert not re.search(pattern, combined), pattern +def test_cloudsql_runtime_requires_the_exact_scoped_identity(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + monkeypatch.delenv("PGPASSWORD", raising=False) + + with pytest.raises(SystemExit, match="requires database user leoclean_kb_runtime"): + module.validate_runtime_connection(SimpleNamespace(credential_mode="runtime", user=None, password_secret=None)) + with pytest.raises(SystemExit, match="requires database user leoclean_kb_runtime"): + module.validate_runtime_connection( + SimpleNamespace( + credential_mode="runtime", + user="postgres", + password_secret=module.RUNTIME_PASSWORD_SECRET, + ) + ) + with pytest.raises(SystemExit, match="requires database user leoclean_kb_runtime"): + module.validate_runtime_connection( + SimpleNamespace( + credential_mode="runtime", + user=f" {module.RUNTIME_DB_USER}", + password_secret=module.RUNTIME_PASSWORD_SECRET, + ) + ) + with pytest.raises(SystemExit, match="requires scoped password secret"): + module.validate_runtime_connection( + SimpleNamespace( + credential_mode="runtime", + user=module.RUNTIME_DB_USER, + password_secret="gcp-teleo-pgvector-standby-postgres-password", + ) + ) + with pytest.raises(SystemExit, match="requires scoped password secret"): + module.validate_runtime_connection( + SimpleNamespace(credential_mode="runtime", user=module.RUNTIME_DB_USER, password_secret=None) + ) + + +def test_cloudsql_runtime_accepts_only_scoped_credentials_and_rejects_inherited_password( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = SimpleNamespace( + credential_mode="runtime", + user=module.RUNTIME_DB_USER, + password_secret=module.RUNTIME_PASSWORD_SECRET, + ) + monkeypatch.delenv("PGPASSWORD", raising=False) + + module.validate_runtime_connection(args) + + monkeypatch.setenv("PGPASSWORD", "") + with pytest.raises(SystemExit, match="Refusing inherited PGPASSWORD"): + module.validate_runtime_connection(args) + + +def test_cloudsql_clone_mode_is_read_only_and_bound_to_one_disposable_database( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = SimpleNamespace( + credential_mode="clone-readonly", + user="postgres", + password_secret=None, + canonical_db="teleo_clone_canary_123", + db="teleo_clone_canary_123", + command="status", + ) + monkeypatch.setenv("PGPASSWORD", "clone-only-password") + monkeypatch.setenv("PGOPTIONS", module.CLONE_READ_ONLY_PGOPTIONS) + + module.validate_runtime_connection(args) + assert module.password(args) == "clone-only-password" + + args.command = "propose-core-change" + with pytest.raises(SystemExit, match="read commands only"): + module.validate_runtime_connection(args) + args.command = "status" + args.canonical_db = "teleo_canonical" + with pytest.raises(SystemExit, match=r"teleo_clone_\*"): + module.validate_runtime_connection(args) + + +def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = SimpleNamespace( + credential_mode="runtime", + user=module.RUNTIME_DB_USER, + password_secret=module.RUNTIME_PASSWORD_SECRET, + host="10.61.0.3", + port="5432", + db="teleo_canonical", + canonical_db="teleo_canonical", + project="teleo-501523", + command="status", + ) + monkeypatch.delenv("PGPASSWORD", raising=False) + monkeypatch.setenv("PGHOST", "attacker.invalid") + monkeypatch.setenv("PGSERVICE", "untrusted-service") + monkeypatch.setenv("PGPASSFILE", "/tmp/untrusted-pgpass") + monkeypatch.setenv("PGOPTIONS", "-c search_path=attacker") + captured: dict[str, object] = {} + + def fake_run(command, **kwargs): + if command[0] == "gcloud": + return SimpleNamespace(returncode=0, stdout="scoped-runtime-password\n", stderr="") + captured["command"] = command + captured["kwargs"] = kwargs + return SimpleNamespace(returncode=0, stdout="ok\n", stderr="") + + monkeypatch.setattr(module.subprocess, "run", fake_run) + + assert module.run_psql(args, "select 1;") == "ok\n" + assert captured["command"] == ["psql", "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"] + kwargs = captured["kwargs"] + assert isinstance(kwargs, dict) + env = kwargs["env"] + assert env["PGHOST"] == "10.61.0.3" + assert env["PGPORT"] == "5432" + assert env["PGDATABASE"] == "teleo_canonical" + assert env["PGUSER"] == module.RUNTIME_DB_USER + assert env["PGSSLMODE"] == "require" + assert env["PGCONNECT_TIMEOUT"] == "8" + assert env["PGPASSWORD"] == "scoped-runtime-password" + assert "PGOPTIONS" not in env + assert "PGSERVICE" not in env + assert "PGPASSFILE" not in env + + +def test_cloudsql_proposal_calls_leave_leo_identity_to_the_database(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + captured: list[str] = [] + + def fake_psql_json_lines(_args, sql, db=None): + captured.append(sql) + return [{"id": "proposal-id", "database": db}] + + monkeypatch.setattr(module, "psql_json_lines", fake_psql_json_lines) + core_args = SimpleNamespace( + proposal_type="revise_claim", + target_kind="claim", + target_ref="claim-id", + current="old", + proposed="new", + evidence=[], + implication=[], + originator="", + channel="telegram", + source_ref="telegram:1", + rationale="Grounded correction", + canonical_db="teleo_canonical", + ) + edge_args = SimpleNamespace( + from_claim="11111111-1111-4111-8111-111111111111", + to_claim="22222222-2222-4222-8222-222222222222", + edge_type="supports", + channel="telegram", + source_ref="telegram:2", + rationale="Grounded relation", + canonical_db="teleo_canonical", + ) + + module.propose_core_change(core_args) + module.propose_edge(edge_args) + + assert "p.proposed_by_handle" not in captured[0] + assert "as proposed_by_handle" not in captured[0] + assert "p.proposed_by_handle" not in captured[1] + assert "as proposed_by_handle" not in captured[1] + assert "stage_leoclean_proposal(\n p.proposal_type,\n p.channel," in captured[0] + assert "stage_leoclean_proposal(\n 'add_edge',\n p.channel," in captured[1] + + +def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None: + sql = GCP_RUNTIME_ROLE_SQL.read_text() + + assert "security definer" in sql.lower() + assert "stage_leoclean_proposal" in sql + assert "'pending_review'" in sql + assert "revoke all privileges on all tables in schema public, kb_stage" in sql + assert "can_insert_proposals_directly" in sql + assert "can_create_database_objects" in sql + assert "can_create_temp_objects" in sql + assert "unexpectedly has database CREATE" in sql + assert "REVOKE TEMP FROM PUBLIC would be a" in sql + assert "can execute unexpected SECURITY DEFINER functions" in sql + assert "revoke all privileges (%s) on table %I.%I from %I" in sql + assert re.search( + r"grant\s+insert\s*\([^)]*proposal_type[^)]*payload[^)]*\)\s+" + r"on\s+kb_stage\.kb_proposals\s+to\s+leoclean_kb_stage_owner", + sql, + re.IGNORECASE | re.DOTALL, + ) + assert not re.search( + r"grant\s+insert(?:\s*\([^)]*\))?\s+on\s+[^;]+\s+to\s+leoclean_kb_runtime", + sql, + re.IGNORECASE | re.DOTALL, + ) + assert "grant update" not in sql.lower() + assert "grant delete" not in sql.lower() + + def _load_module(path: Path): spec = importlib.util.spec_from_file_location(path.stem, path) assert spec and spec.loader @@ -323,11 +569,16 @@ def test_cloudsql_status_works_without_optional_audit_fallback(monkeypatch) -> N assert all("teleo_restore.response_audit" not in sql or "to_regclass" in sql for sql, _db in calls) -def test_cloudsql_zero_hit_search_stays_canonical_when_audit_is_unavailable(monkeypatch) -> None: +def test_cloudsql_zero_hit_search_stays_canonical_when_audit_is_disabled(monkeypatch) -> None: module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") canonical = {"artifact": "teleo_cloudsql_canonical_kb_context", "hit_count_total": 0, "hits": []} monkeypatch.setattr(module, "query_canonical_rows", lambda *_args: canonical.copy()) - monkeypatch.setattr(module, "audit_restore_available", lambda _args: False) + monkeypatch.delenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", raising=False) + monkeypatch.setattr( + module, + "audit_restore_available", + lambda _args: (_ for _ in ()).throw(AssertionError("audit availability must not be probed")), + ) monkeypatch.setattr( module, "query_audit_rows", @@ -337,7 +588,22 @@ def test_cloudsql_zero_hit_search_stays_canonical_when_audit_is_unavailable(monk result = module.query_rows(SimpleNamespace(), "unseen query", 8) assert result["hits"] == [] - assert "audit fallback unavailable" in result["canonical_fallback_reason"] + assert "audit fallback disabled" in result["canonical_fallback_reason"] + + +def test_cloudsql_zero_hit_search_uses_audit_only_when_explicitly_enabled(monkeypatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + canonical = {"artifact": "teleo_cloudsql_canonical_kb_context", "hit_count_total": 0, "hits": []} + audit = {"artifact": "teleo_cloudsql_memory_context", "hits": [{"row_id": "audit-1"}]} + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "1") + monkeypatch.setattr(module, "query_canonical_rows", lambda *_args: canonical.copy()) + monkeypatch.setattr(module, "audit_restore_available", lambda _args: True) + monkeypatch.setattr(module, "query_audit_rows", lambda *_args: audit.copy()) + + result = module.query_rows(SimpleNamespace(), "unseen query", 8) + + assert result["hits"] == [{"row_id": "audit-1"}] + assert result["canonical_fallback_reason"] == "no matching canonical public/persona/strategy/belief rows" def test_vps_bridge_verifies_relative_source_artifact_against_canonical_hash(tmp_path: Path) -> None: diff --git a/tests/test_teleo_agent_systemd.py b/tests/test_teleo_agent_systemd.py index 154ac86..99c0535 100644 --- a/tests/test_teleo_agent_systemd.py +++ b/tests/test_teleo_agent_systemd.py @@ -41,7 +41,7 @@ def test_auto_deploy_prefers_github_remote_when_present(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() assert 'DEPLOY_REMOTE="${TELEO_DEPLOY_REMOTE:-}"' in auto_deploy - assert 'remote get-url github' in auto_deploy + assert "remote get-url github" in auto_deploy assert 'git fetch "$DEPLOY_REMOTE" main' in auto_deploy assert 'git rev-parse "$DEPLOY_REMOTE/main"' in auto_deploy assert 'git merge --ff-only "$DEPLOY_REMOTE/main"' in auto_deploy @@ -50,7 +50,7 @@ def test_auto_deploy_prefers_github_remote_when_present(): def test_auto_deploy_executable_safety_net_stays_inside_synced_dirs(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() - assert "for dir in \"$PIPELINE_DIR\" \"$TELEGRAM_DIR\" \"$DIAGNOSTICS_DIR\" \"$AGENT_STATE_DIR\"" in auto_deploy + assert 'for dir in "$PIPELINE_DIR" "$TELEGRAM_DIR" "$DIAGNOSTICS_DIR" "$AGENT_STATE_DIR"' in auto_deploy assert "find /opt/teleo-eval -maxdepth 3 -name '*.sh'" not in auto_deploy assert "backups" in auto_deploy @@ -61,7 +61,7 @@ def test_agent_healthcheck_timer_runs_both_live_telegram_agents(): assert "/opt/teleo-eval/telegram/agent_healthcheck.py" in service assert "--agents leo leo-wallet-test" in service - assert "--since \"20 min ago\"" in service + assert '--since "20 min ago"' in service assert "OnUnitActiveSec=5min" in timer assert "WantedBy=timers.target" in timer @@ -74,7 +74,7 @@ def test_deploy_scripts_install_systemd_units_and_enable_agent_healthcheck(): assert 'sudo install -m 0644 "$unit" "$SYSTEMD_DIR/$(basename "$unit")"' in auto_deploy assert "sudo systemctl daemon-reload" in auto_deploy assert "sudo systemctl enable --now teleo-agent-healthcheck.timer" in auto_deploy - assert "git diff --name-only \"$OLD_SHA\" \"$NEW_SHA\" -- systemd/teleo-agent@.service" in auto_deploy + assert 'git diff --name-only "$OLD_SHA" "$NEW_SHA" -- systemd/teleo-agent@.service' in auto_deploy assert 'VPS_SYSTEMD="/etc/systemd/system"' in manual_deploy assert "teleo-agent-healthcheck.timer" in manual_deploy @@ -100,16 +100,16 @@ def test_deploy_scripts_sync_leoclean_skills_and_restart_gateway_for_runtime_cha assert 'LEOCLEAN_BIN_DIR="/home/teleo/.hermes/profiles/leoclean/bin"' in auto_deploy assert 'LEOCLEAN_SKILLS_DIR="/home/teleo/.hermes/profiles/leoclean/skills"' in auto_deploy assert 'LEOCLEAN_PLUGINS_DIR="/home/teleo/.hermes/profiles/leoclean/plugins"' in auto_deploy - assert 'hermes-agent/leoclean-bin/' in auto_deploy - assert 'hermes-agent/leoclean-skills/vps/' in auto_deploy - assert 'hermes-agent/leoclean-plugins/vps/' in auto_deploy + assert "hermes-agent/leoclean-bin/" in auto_deploy + assert "hermes-agent/leoclean-skills/vps/" in auto_deploy + assert "hermes-agent/leoclean-plugins/vps/" in auto_deploy assert 'HERMES_PATCH_DIR="/home/teleo/.hermes/teleo-runtime-patches"' in auto_deploy assert '"status": "installed_now"' in auto_deploy - assert 'Hermes response-transform drift repaired' in auto_deploy - assert 'deploy/leoclean-gateway-restart-required.sh' in auto_deploy - assert 'TELEO_AUTO_DEPLOY_REEXECED' in auto_deploy + assert "Hermes response-transform drift repaired" in auto_deploy + assert "deploy/leoclean-gateway-restart-required.sh" in auto_deploy + assert "TELEO_AUTO_DEPLOY_REEXECED" in auto_deploy assert 'exec bash "$DEPLOY_CHECKOUT/deploy/auto-deploy.sh"' in auto_deploy - assert 'add_restart_if_unit_active leoclean-gateway' in auto_deploy + assert "add_restart_if_unit_active leoclean-gateway" in auto_deploy assert 'sudo systemctl restart "$svc"' in auto_deploy assert "sudo systemctl restart $RESTART" not in auto_deploy assert "NOPASSWD: /usr/bin/systemctl restart leoclean-gateway" in sudoers @@ -117,11 +117,11 @@ def test_deploy_scripts_sync_leoclean_skills_and_restart_gateway_for_runtime_cha assert 'VPS_LEOCLEAN_BIN="/home/teleo/.hermes/profiles/leoclean/bin"' in manual_deploy assert 'VPS_LEOCLEAN_SKILLS="/home/teleo/.hermes/profiles/leoclean/skills"' in manual_deploy assert 'VPS_LEOCLEAN_PLUGINS="/home/teleo/.hermes/profiles/leoclean/plugins"' in manual_deploy - assert 'hermes-agent/leoclean-bin/' in manual_deploy - assert 'hermes-agent/leoclean-skills/vps/' in manual_deploy - assert 'hermes-agent/leoclean-plugins/vps/' in manual_deploy + assert "hermes-agent/leoclean-bin/" in manual_deploy + assert "hermes-agent/leoclean-skills/vps/" in manual_deploy + assert "hermes-agent/leoclean-plugins/vps/" in manual_deploy assert 'VPS_HERMES_PATCHES="/home/teleo/.hermes/teleo-runtime-patches"' in manual_deploy - assert 'systemctl is-active --quiet leoclean-gateway.service' in manual_deploy + assert "systemctl is-active --quiet leoclean-gateway.service" in manual_deploy def test_deploy_scripts_syntax_check_leoclean_database_runtime_before_restart(): @@ -135,32 +135,22 @@ def test_deploy_scripts_syntax_check_leoclean_database_runtime_before_restart(): def test_auto_deploy_hot_syncs_leoclean_markdown_without_gateway_restart(): - assert not _leoclean_gateway_restart_required( - "hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md" - ) - assert not _leoclean_gateway_restart_required( - "hermes-agent/leoclean-skills/vps/live-leo-telegram/SKILL.md" - ) + assert not _leoclean_gateway_restart_required("hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md") + assert not _leoclean_gateway_restart_required("hermes-agent/leoclean-skills/vps/live-leo-telegram/SKILL.md") def test_auto_deploy_restarts_gateway_for_leoclean_runtime_changes(): assert _leoclean_gateway_restart_required("hermes-agent/leoclean-bin/kb_tool.py") - assert _leoclean_gateway_restart_required( - "hermes-agent/leoclean-skills/vps/teleo-kb-bridge/runtime_helper.py" - ) - assert _leoclean_gateway_restart_required( - "hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py" - ) - assert _leoclean_gateway_restart_required( - "hermes-agent/patches/apply_response_transform_hook.py" - ) + assert _leoclean_gateway_restart_required("hermes-agent/leoclean-skills/vps/teleo-kb-bridge/runtime_helper.py") + assert _leoclean_gateway_restart_required("hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py") + assert _leoclean_gateway_restart_required("hermes-agent/patches/apply_response_transform_hook.py") def test_auto_deploy_sudoers_installer_is_root_only_and_visudo_checked(): installer = (REPO_ROOT / "deploy" / "install-auto-deploy-sudoers.sh").read_text() assert 'TARGET="/etc/sudoers.d/teleo-auto-deploy"' in installer - assert 'id -u' in installer + assert "id -u" in installer assert 'install -m 0440 "$SOURCE" "$TARGET"' in installer assert 'visudo -cf "$TARGET"' in installer @@ -183,3 +173,45 @@ def test_gcp_leoclean_skill_sync_targets_parallel_runtime_without_secrets(): assert "sudo rsync" not in script assert "TELEGRAM_BOT_TOKEN" not in script assert "CLOUDSQL_PASSWORD" not in script + + +def test_gcp_runtime_sync_is_source_bound_and_scoped(): + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + dropin = (REPO_ROOT / "systemd" / "leoclean-gcp-prod-parallel-cloudsql.conf").read_text() + + assert 'git -C "$REPO_ROOT" rev-parse HEAD' in script + assert 'git -C "$REPO_ROOT" diff --quiet HEAD' in script + assert "merge-base --is-ancestor" in script + assert "Refusing to deploy unmerged revision" in script + assert "Refusing to install runtime files without --restart" in script + assert 'DROPIN_NAME="10-cloudsql-memory.conf"' in script + assert "20-canonical-kb.conf" not in script + assert "sha256sum" in script + assert "--restart" in script + assert 'python3 "$tool_path" status --format json --redacted' in script + assert "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" in script + assert "metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" in script + assert "env -u PGPASSWORD" in script + assert 'run_scoped_status pre "$remote_tmp/$TOOL_REL"' in script + assert 'run_scoped_status post "$profile_bin/cloudsql_memory_tool.py"' in script + assert "backup_path" in script + assert "restore_path" in script + assert "rollback_runtime" in script + assert "trap 'on_exit $?' EXIT" in script + assert 'backup_path "$dropin_dir/$DROPIN_NAME" dropin' in script + assert 'dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$"' in script + assert 'sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME"' in script + assert 'active_state" != "active"' in script + assert 'sub_state" != "running"' in script + assert 'sudo systemctl stop "$SERVICE"' in script + assert 'sudo systemctl start "$SERVICE"' in script + assert "UnsetEnvironment=PGPASSWORD" in dropin + assert "RuntimeDirectory=leoclean-gcp-prod-parallel" in dropin + assert "RuntimeDirectoryMode=0700" in dropin + assert "Environment=CLOUDSDK_CONFIG=/run/leoclean-gcp-prod-parallel" in dropin + assert "TELEO_KB_MODE=cloudsql" in dropin + assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in dropin + assert "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" in dropin + assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in dropin + assert not any(line.startswith("Environment=PGPASSWORD") for line in dropin.splitlines()) + assert "gcp-teleo-pgvector-standby-postgres-password" not in dropin From b7bdcea46acdf9ddff85c56ce6bda3837a576752 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 04:40:26 +0200 Subject: [PATCH 2/7] fix: enforce scoped GCP Leo runtime credentials --- .github/workflows/ci.yml | 7 + deploy/sync-gcp-leoclean-runtime.sh | 696 ++++++- docs/gcp-leoclean-runtime-reconciliation.md | 144 +- .../leoclean-bin/cloudsql_memory_tool.py | 291 ++- hermes-agent/leoclean-bin/teleo-kb | 37 +- ops/gcp_leoclean_runtime_role.sql | 53 +- ...verify_gcp_leoclean_runtime_permissions.py | 1838 +++++++++++++++++ ...verify_gcp_leoclean_service_environment.py | 219 ++ ...run_gcp_generated_db_direct_claim_suite.py | 2 +- .../leoclean-gcp-prod-parallel-cloudsql.conf | 16 +- .../test_gcp_leoclean_runtime_permissions.py | 797 +++++++ .../test_gcp_leoclean_service_environment.py | 281 +++ .../test_hermes_leoclean_kb_bridge_source.py | 523 ++++- tests/test_teleo_agent_systemd.py | 351 +++- 14 files changed, 5031 insertions(+), 224 deletions(-) create mode 100644 ops/verify_gcp_leoclean_runtime_permissions.py create mode 100755 ops/verify_gcp_leoclean_service_environment.py create mode 100644 tests/test_gcp_leoclean_runtime_permissions.py create mode 100644 tests/test_gcp_leoclean_service_environment.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91dada8..3367127 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,8 @@ jobs: ops/restore_gcp_generated_postgres_snapshot.py \ ops/sqlite_to_postgres_dump.py \ ops/verify_gcp_cloudsql_restore_readback.py \ + ops/verify_gcp_leoclean_runtime_permissions.py \ + ops/verify_gcp_leoclean_service_environment.py \ ops/verify_postgres_parity_manifest.py \ telegram/approvals.py \ hermes-agent/leoclean-bin/kb_tool.py \ @@ -80,6 +82,8 @@ jobs: tests/test_capture_vps_canonical_postgres_snapshot.py \ tests/test_gcp_infra_execute_canary.py \ tests/test_gcp_infra_readiness_checker.py \ + tests/test_gcp_leoclean_runtime_permissions.py \ + tests/test_gcp_leoclean_service_environment.py \ tests/test_gcp_runtime_baseline_apply.py \ tests/test_gcp_service_communications.py \ tests/test_gcp_cloudsql_restore_drill.py \ @@ -102,12 +106,15 @@ jobs: tests/test_restore_gcp_generated_postgres_snapshot.py \ tests/test_sqlite_to_postgres_dump.py \ tests/test_sqlite_postgres_restore_canary_capsule.py \ + tests/test_teleo_agent_systemd.py \ tests/test_eval_parse.py \ tests/test_contributor.py \ tests/test_search.py - name: Shell syntax run: | bash -n \ + deploy/sync-gcp-leoclean-runtime.sh \ + hermes-agent/leoclean-bin/teleo-kb \ ops/backup_vps_sqlite_kb.sh \ ops/run_gcp_cloudsql_restore_drill.sh \ ops/run_sqlite_postgres_restore_canary.sh diff --git a/deploy/sync-gcp-leoclean-runtime.sh b/deploy/sync-gcp-leoclean-runtime.sh index 3661155..25a7976 100755 --- a/deploy/sync-gcp-leoclean-runtime.sh +++ b/deploy/sync-gcp-leoclean-runtime.sh @@ -10,23 +10,28 @@ ZONE="${ZONE:-europe-west6-a}" INSTANCE="${INSTANCE:-teleo-prod-1}" SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}" MERGED_REF="${MERGED_REF:-origin/main}" -REMOTE_PROFILE_BIN="${REMOTE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}" +REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}" +SERVICE_PROFILE_BIN="${SERVICE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}" +SERVICE_PROFILE_ROOT="${SERVICE_PROFILE_ROOT:-/home/teleo/.hermes/profiles/leoclean}" +SERVICE_HERMES_ROOT="${SERVICE_HERMES_ROOT:-/home/teleo/.hermes}" REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}" -REMOTE_CLOUDSDK_CONFIG="${REMOTE_CLOUDSDK_CONFIG:-/run/leoclean-gcp-prod-parallel}" EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}" WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb" TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py" +PERMISSION_VERIFIER_REL="ops/verify_gcp_leoclean_runtime_permissions.py" +SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py" DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf" DROPIN_NAME="10-cloudsql-memory.conf" DRY_RUN=false RESTART=false VERIFY_ONLY=false +PREFLIGHT_ONLY=false usage() { cat <<'USAGE' -Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--restart] [--verify-only] +Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--preflight-only] [--restart] [--verify-only] The database role and scoped Secret Manager secret must exist before --restart. This script deploys only the bridge files and non-secret systemd drop-in. @@ -36,6 +41,7 @@ USAGE for arg in "$@"; do case "$arg" in --dry-run) DRY_RUN=true ;; + --preflight-only) PREFLIGHT_ONLY=true ;; --restart) RESTART=true ;; --verify-only) VERIFY_ONLY=true ;; --help|-h) @@ -50,28 +56,72 @@ for arg in "$@"; do esac done -for path in "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; do +if $PREFLIGHT_ONLY && $RESTART; then + echo "--preflight-only and --restart are mutually exclusive." >&2 + exit 64 +fi +if $VERIFY_ONLY && { $PREFLIGHT_ONLY || $RESTART; }; then + echo "--verify-only cannot be combined with --preflight-only or --restart." >&2 + exit 64 +fi + +for path in \ + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$DROPIN_REL"; do if [ ! -f "$REPO_ROOT/$path" ]; then echo "Required source file is missing: $path" >&2 exit 66 fi + if [ -L "$REPO_ROOT/$path" ]; then + echo "Refusing symlinked runtime source file: $path" >&2 + exit 65 + fi done if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2 exit 65 fi -if ! grep -Fxq "Environment=CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG" "$REPO_ROOT/$DROPIN_REL"; then - echo "Runtime drop-in CLOUDSDK_CONFIG does not match $REMOTE_CLOUDSDK_CONFIG." >&2 +if ! grep -Fxq 'Environment=TELEO_GCP_METADATA_ONLY=1' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must enforce metadata-only GCP credentials." >&2 + exit 65 +fi +if ! grep -Fxq "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not bind the root-controlled runtime into the service profile." >&2 + exit 65 +fi +if ! grep -Fxq "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not protect the service-profile ancestor from replacement." >&2 + exit 65 +fi +if ! grep -Fxq "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not limit service-profile writes to state and workspace." >&2 + exit 65 +fi +if ! grep -Fxq 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must remove CAP_SYS_ADMIN from the service." >&2 exit 65 fi SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)" -if ! git -C "$REPO_ROOT" diff --quiet HEAD -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; then +if ! git -C "$REPO_ROOT" diff --quiet HEAD -- \ + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$DROPIN_REL"; then echo "Refusing to deploy runtime files that do not match HEAD." >&2 exit 65 fi -if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL")" ]; then +if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- \ + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$DROPIN_REL")" ]; then echo "Refusing to deploy untracked runtime files." >&2 exit 65 fi @@ -85,6 +135,8 @@ if ! git -C "$REPO_ROOT" merge-base --is-ancestor "$SOURCE_REVISION" "$MERGED_RE fi WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')" TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')" +PERMISSION_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" +SERVICE_ENV_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')" GCP_SSH=( @@ -123,64 +175,132 @@ assert_service_running() { fi } -assert_runtime_environment() { - local environment runtime_dir_state - environment="$(systemctl show "$SERVICE" --property=Environment --value)" - case " $environment " in - *" PGPASSWORD="*) - echo "Service environment still contains PGPASSWORD." >&2 +assert_unit_configuration() { + local require_reviewed="${1:-true}" + local dropin_path dropin_name dropin_paths environment_files reviewed_dropin_seen=false + dropin_paths="$(systemctl show "$SERVICE" --property=DropInPaths --value)" + environment_files="$(systemctl show "$SERVICE" --property=EnvironmentFiles --value)" + if [ -n "$environment_files" ]; then + echo "Service has an effective EnvironmentFile, which is not allowed for the scoped runtime." >&2 + return 1 + fi + for dropin_path in $dropin_paths; do + if [ "$dropin_path" = "$REMOTE_DROPIN_DIR/$DROPIN_NAME" ]; then + reviewed_dropin_seen=true + continue + fi + dropin_name="${dropin_path##*/}" + if [[ "$dropin_name" > "$DROPIN_NAME" ]]; then + echo "Service has a later drop-in than the reviewed Cloud SQL configuration: $dropin_name" >&2 return 1 - ;; - esac - case " $environment " in - *" CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG "*) ;; - *) - echo "Service does not use the isolated CLOUDSDK_CONFIG." >&2 - return 1 - ;; - esac - runtime_dir_state="$(stat -c '%U:%G:%a' "$REMOTE_CLOUDSDK_CONFIG")" - if [ "$runtime_dir_state" != "teleo:teleo:700" ]; then - echo "Isolated CLOUDSDK_CONFIG has unexpected ownership or mode: $runtime_dir_state" >&2 + fi + done + if [ "$require_reviewed" = true ] && [ "$reviewed_dropin_seen" != true ]; then + echo "Service does not load the reviewed Cloud SQL drop-in." >&2 return 1 fi } +assert_runtime_environment() { + local verifier_path="$1" require_reviewed="${2:-true}" main_pid + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + sudo /usr/bin/python3 "$verifier_path" --pid "$main_pid" + assert_unit_configuration "$require_reviewed" +} + assert_attached_service_account() { - local sdk_config="$1" metadata_email - sudo install -d -o teleo -g teleo -m 0700 "$sdk_config" + local metadata_email metadata_email="$( - sudo -n -u teleo env -u PGPASSWORD \ + sudo -n -u teleo env -i \ HOME=/home/teleo \ - CLOUDSDK_CONFIG="$sdk_config" \ - curl --fail --silent --show-error \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + /usr/bin/curl -q --noproxy '*' --proto '=http' --max-time 10 --fail --silent --show-error \ --header 'Metadata-Flavor: Google' \ - 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email' + 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email' )" if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then echo "Attached VM service account mismatch: $metadata_email" >&2 return 1 fi - - # An empty, phase-local Cloud SDK config forces this token request to use - # the VM's attached identity instead of an operator's cached credentials. - sudo -n -u teleo env -u PGPASSWORD \ - HOME=/home/teleo \ - CLOUDSDK_CONFIG="$sdk_config" \ - CLOUDSDK_CORE_PROJECT="$PROJECT" \ - gcloud --quiet auth print-access-token >/dev/null echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email" } +assert_administrator_secret_denied() { + local main_pid + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + sudo nsenter -t "$main_pid" -m --env -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --init-groups \ + --no-new-privs \ + /usr/bin/python3 - "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" <<'PY' +import json +import sys +import urllib.error +import urllib.parse +import urllib.request + +project, expected_email = sys.argv[1:] +metadata_headers = {"Metadata-Flavor": "Google"} +opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def metadata(path: str) -> bytes: + request = urllib.request.Request( + "http://169.254.169.254/computeMetadata/v1/" + path, + headers=metadata_headers, + ) + with opener.open(request, timeout=10) as response: + return response.read(65536) + + +try: + email = metadata("instance/service-accounts/default/email").decode("utf-8").strip() + if email != expected_email: + raise RuntimeError + token_payload = json.loads(metadata("instance/service-accounts/default/token")) + token = token_payload.get("access_token") + if token_payload.get("token_type") != "Bearer" or not isinstance(token, str) or not token: + raise RuntimeError + secret = urllib.parse.quote("gcp-teleo-pgvector-standby-postgres-password", safe="") + url = f"https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{secret}/versions/latest:access" + request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + try: + response = opener.open(request, timeout=10) + except urllib.error.HTTPError as error: + body = error.read(65536) + parsed = json.loads(body) + detail = parsed.get("error", {}) + message = str(detail.get("message", "")).lower() + if error.code != 403 or detail.get("status") != "PERMISSION_DENIED": + raise RuntimeError from None + if "secretmanager.versions.access" not in message: + raise RuntimeError + else: + response.close() + raise RuntimeError +except Exception: + raise SystemExit("administrator secret was not the exact expected metadata-identity IAM denial") from None + +print(json.dumps({ + "administrator_secret_access": "iam_permission_denied", + "credential_context": "systemd_main_pid_environment", + "identity_source": "gce_metadata", + "stdout_discarded": True, +}, sort_keys=True)) +PY +} + run_scoped_status() { - local phase="$1" tool_path="$2" sdk_config="$3" - assert_attached_service_account "$sdk_config" + local phase="$1" tool_path="$2" + assert_attached_service_account echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical" - sudo -n -u teleo env -u PGPASSWORD \ + sudo -n -u teleo env -i \ HOME=/home/teleo \ - CLOUDSDK_CONFIG="$sdk_config" \ - CLOUDSDK_CORE_PROJECT="$PROJECT" \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ TELEO_GCP_PROJECT=teleo-501523 \ + TELEO_GCP_METADATA_ONLY=1 \ TELEO_CLOUDSQL_HOST=10.61.0.3 \ TELEO_CLOUDSQL_PORT=5432 \ TELEO_CLOUDSQL_DB=teleo_canonical \ @@ -188,46 +308,238 @@ run_scoped_status() { TELEO_CLOUDSQL_USER=leoclean_kb_runtime \ TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \ TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \ - python3 "$tool_path" status --format json --redacted + /usr/bin/python3 "$tool_path" status --format json --redacted +} + +run_service_wrapper_status() { + local phase="$1" main_pid + assert_attached_service_account + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + echo "SCOPED_WRAPPER_STATUS phase=$phase user=teleo database=teleo_canonical" + # --env imports the target MainPID's effective /proc environment. setpriv + # drops back to the service's Unix identity without sudo resetting that env. + sudo nsenter -t "$main_pid" -m --env -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --init-groups \ + --no-new-privs \ + "$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted +} + +service_fingerprint() { + printf '%s|%s|%s|%s' \ + "$(systemctl show "$SERVICE" --property=ActiveState --value)" \ + "$(systemctl show "$SERVICE" --property=SubState --value)" \ + "$(systemctl show "$SERVICE" --property=MainPID --value)" \ + "$(systemctl show "$SERVICE" --property=NRestarts --value)" +} + +assert_service_fingerprint() { + local expected="$1" actual + actual="$(service_fingerprint)" + if [ "$actual" != "$expected" ]; then + echo "Service state changed during the read-only preflight." >&2 + return 1 + fi + echo "PREFLIGHT_SERVICE_UNCHANGED fingerprint=$actual" +} + +run_permission_verifier() { + local phase="$1" verifier_path="$2" receipt_path="$3" receipt_dir run_id receipt_state + run_id="$(date -u +%Y%m%d%H%M%S)-$phase" + receipt_dir="$(dirname "$receipt_path")" + sudo -n -u teleo mkdir -p -- "$receipt_dir" + sudo -n -u teleo chmod 0700 "$receipt_dir" + echo "NEGATIVE_PERMISSION_RECEIPT phase=$phase run_id=$run_id" + sudo -n -u teleo env -i \ + HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + /usr/bin/python3 "$verifier_path" --run-id "$run_id" --output "$receipt_path" + receipt_state="$(sudo stat -c '%U:%G:%a' "$receipt_path")" + if [ "$receipt_state" != "teleo:teleo:600" ]; then + echo "Permission receipt has unexpected ownership or mode: $receipt_state" >&2 + return 1 + fi + sudo /usr/bin/python3 - "$receipt_path" <<'PY' +import json +import sys +from pathlib import Path + +receipt = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +expected = { + "artifact": "gcp_leoclean_runtime_permissions", + "current_tier": "T3_live_readonly", + "mode": "live_private_gcp_staging", + "required_tier": "T3_live_readonly", + "status": "pass", +} +mismatched = sorted(key for key, value in expected.items() if receipt.get(key) != value) +if mismatched: + raise SystemExit("permission receipt contract mismatch: " + ",".join(mismatched)) +print(json.dumps({"permission_receipt": "validated", "validated_fields": sorted(expected)}, sort_keys=True)) +PY +} + +assert_installed_files() { + local wrapper tool permission_verifier service_env_verifier dropin revision executable_dir protected_dir protected_file + wrapper="$REMOTE_RUNTIME_BIN/teleo-kb" + tool="$REMOTE_RUNTIME_BIN/cloudsql_memory_tool.py" + permission_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" + service_env_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" + dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME" + revision="$REMOTE_RUNTIME_BIN/.livingip-runtime-revision" + test "$(sudo sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA" + test "$(sudo sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA" + test "$(sudo sha256sum "$permission_verifier" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" + test "$(sudo sha256sum "$service_env_verifier" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" + test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA" + test "$(sudo stat -c '%U:%G:%a' "$REMOTE_RUNTIME_BIN")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$wrapper")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$tool")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$permission_verifier")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$service_env_verifier")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$revision")" = "root:root:644" + test "$(sudo stat -c '%U:%G:%a' "$dropin")" = "root:root:644" + test "$(sudo cat "$revision")" = "$SOURCE_REVISION" + for protected_dir in \ + /usr \ + /usr/local \ + /usr/local/libexec \ + /usr/local/libexec/livingip \ + "$REMOTE_RUNTIME_BIN" \ + "$REMOTE_DROPIN_DIR"; do + sudo test -d "$protected_dir" + sudo test ! -L "$protected_dir" + test "$(sudo stat -c '%U:%G' "$protected_dir")" = "root:root" + sudo -n -u teleo test ! -w "$protected_dir" + done + for protected_file in \ + "$wrapper" \ + "$tool" \ + "$permission_verifier" \ + "$service_env_verifier" \ + "$revision" \ + "$dropin"; do + sudo test -f "$protected_file" + sudo test ! -L "$protected_file" + sudo -n -u teleo test ! -w "$protected_file" + done + for executable_dir in \ + /usr/local/sbin \ + /usr/local/bin \ + /usr/sbin \ + /usr/bin \ + /sbin \ + /bin; do + sudo test -d "$executable_dir" + test "$(sudo stat -Lc '%U:%G' "$executable_dir")" = "root:root" + sudo -n -u teleo test ! -w "$executable_dir" + done + for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3; do + sudo test -x "$protected_file" + test "$(sudo stat -Lc '%U:%G' "$protected_file")" = "root:root" + sudo -n -u teleo test ! -w "$protected_file" + done + sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin" + sudo grep -E '^Environment=TELEO_(KB_MODE|GCP_PROJECT|GCP_METADATA_ONLY|CLOUDSQL_|CANONICAL_CLOUDSQL_DB)' "$dropin" + sudo grep -Fx "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$dropin" + sudo grep -Fx "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$dropin" + sudo grep -Fx "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$dropin" + sudo grep -Fx 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$dropin" } assert_deployed_files() { - local wrapper tool dropin - wrapper="$REMOTE_PROFILE_BIN/teleo-kb" - tool="$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py" - dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME" - test "$(sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA" - test "$(sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA" - test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA" - sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin" - sudo grep -E '^Environment=(CLOUDSDK_CONFIG|TELEO_(KB_MODE|GCP_PROJECT|CLOUDSQL_|CANONICAL_CLOUDSQL_DB))' "$dropin" + local main_pid namespace_state source_state mount_state + local hermes_mount_state state_mount_state workspace_mount_state capability_set + assert_installed_files + + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + source_state="$(stat -Lc '%d:%i' "$REMOTE_RUNTIME_BIN")" + namespace_state="$(sudo nsenter -t "$main_pid" -m -- stat -Lc '%d:%i' "$SERVICE_PROFILE_BIN")" + if [ "$namespace_state" != "$source_state" ]; then + echo "Service namespace does not bind the reviewed runtime directory." >&2 + return 1 + fi + mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -T "$SERVICE_PROFILE_BIN" -n -o OPTIONS)" + case ",$mount_state," in + *,ro,*) ;; + *) + echo "Service runtime bind is not read-only." >&2 + return 1 + ;; + esac + hermes_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_HERMES_ROOT" -n -o OPTIONS)" + state_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/state" -n -o OPTIONS)" + workspace_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/workspace" -n -o OPTIONS)" + case ",$hermes_mount_state," in + *,ro,*) ;; + *) + echo "Service .hermes ancestor mount is not read-only." >&2 + return 1 + ;; + esac + case ",$state_mount_state," in + *,rw,*) ;; + *) + echo "Service state mount is not writable." >&2 + return 1 + ;; + esac + case ",$workspace_mount_state," in + *,rw,*) ;; + *) + echo "Service workspace mount is not writable." >&2 + return 1 + ;; + esac + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w /home + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT/profiles" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_ROOT" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_BIN" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/state" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/workspace" + capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)" + if printf '%s\n' "$capability_set" | grep -Eqi '(^|[[:space:]])cap_sys_admin([[:space:]]|$)'; then + echo "Service capability bounding set still contains CAP_SYS_ADMIN." >&2 + return 1 + fi + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb" | awk '{print $1}')" = "$WRAPPER_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql_memory_tool.py" | awk '{print $1}')" = "$TOOL_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_runtime_permissions.py" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_service_environment.py" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" } REMOTE ) printf -v verify_context \ - 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\n' \ - "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \ - "$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \ - "$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA" + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ + "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ + "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" "$SOURCE_REVISION" verify_body=$(cat <<'REMOTE' set -euo pipefail verification_tmp="$(mktemp -d /tmp/teleo-gcp-runtime-verify.XXXXXX)" +sudo chown teleo:teleo "$verification_tmp" +sudo chmod 0700 "$verification_tmp" cleanup_verification() { sudo rm -rf -- "$verification_tmp"; } trap cleanup_verification EXIT assert_deployed_files assert_service_running -assert_runtime_environment -run_scoped_status verify "$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py" "$verification_tmp/gcloud" +assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" +run_service_wrapper_status verify +assert_administrator_secret_denied +run_permission_verifier verify "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" "$verification_tmp/permission-receipt.json" systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager REMOTE ) verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}" if $DRY_RUN; then - echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE restart=$RESTART" - echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA dropin_sha256=$DROPIN_SHA" + echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE preflight_only=$PREFLIGHT_ONLY restart=$RESTART" + echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA dropin_sha256=$DROPIN_SHA" exit 0 fi @@ -236,39 +548,88 @@ if $VERIFY_ONLY; then exit 0 fi -if ! $RESTART; then - echo "Refusing to install runtime files without --restart; use --dry-run or --verify-only for read-only work." >&2 +if ! $RESTART && ! $PREFLIGHT_ONLY; then + echo "Refusing to install runtime files without --restart; use --dry-run, --preflight-only, or --verify-only for read-only work." >&2 exit 64 fi -REMOTE_TMP="/tmp/teleo-gcp-leoclean-runtime-$(date -u +%Y%m%dT%H%M%SZ)-$$" printf -v sync_context \ - 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\nREMOTE_TMP=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\n' \ - "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \ - "$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \ - "$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA" "$REMOTE_TMP" \ - "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" "$SOURCE_REVISION" + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ + "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ + "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" \ + "$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY" sync_body=$(cat <<'REMOTE' set -euo pipefail -remote_tmp="$REMOTE_TMP" -profile_bin="$REMOTE_PROFILE_BIN" +remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)" +runtime_bin="$REMOTE_RUNTIME_BIN" +runtime_parent="$(dirname "$runtime_bin")" +runtime_grandparent="$(dirname "$runtime_parent")" +runtime_anchor="$(dirname "$runtime_grandparent")" dropin_dir="$REMOTE_DROPIN_DIR" +dropin_parent="$(dirname "$dropin_dir")" +payload_dir="$remote_tmp/payload" backup_dir="$remote_tmp/backup" +work_dir="$remote_tmp/work" +wrapper_stage="$runtime_bin/.teleo-kb.livingip-new.$$" +tool_stage="$runtime_bin/.cloudsql-memory-tool.livingip-new.$$" +permission_verifier_stage="$runtime_bin/.runtime-permission-verifier.livingip-new.$$" +service_env_verifier_stage="$runtime_bin/.service-environment-verifier.livingip-new.$$" +revision_stage="$runtime_bin/.livingip-runtime-revision.livingip-new.$$" dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$" mutation_started=false +assert_existing_root_directory() { + local target="$1" + if sudo test -e "$target" || sudo test -L "$target"; then + sudo test -d "$target" + sudo test ! -L "$target" + test "$(sudo stat -c '%U:%G' "$target")" = "root:root" + sudo -n -u teleo test ! -w "$target" + fi +} + +assert_existing_root_file() { + local target="$1" + if sudo test -e "$target" || sudo test -L "$target"; then + sudo test -f "$target" + sudo test ! -L "$target" + test "$(sudo stat -c '%U:%G' "$target")" = "root:root" + sudo -n -u teleo test ! -w "$target" + fi +} + backup_path() { local target="$1" name="$2" if sudo test -e "$target"; then sudo cp -a -- "$target" "$backup_dir/$name" - touch "$backup_dir/$name.present" + sudo touch "$backup_dir/$name.present" + fi +} + +backup_directory_state() { + local target="$1" name="$2" + if sudo test -d "$target"; then + sudo stat -c '%u %g %a' "$target" | sudo tee "$backup_dir/$name.metadata" >/dev/null + sudo touch "$backup_dir/$name.present" + fi +} + +restore_directory_state() { + local target="$1" name="$2" uid gid mode + if sudo test -f "$backup_dir/$name.present"; then + read -r uid gid mode <<<"$(sudo cat "$backup_dir/$name.metadata")" + sudo chown "$uid:$gid" "$target" + sudo chmod "$mode" "$target" + elif sudo test -d "$target"; then + sudo rmdir -- "$target" fi } restore_path() { local target="$1" name="$2" restore_rc=0 restore_tmp - if [ -f "$backup_dir/$name.present" ]; then + if sudo test -f "$backup_dir/$name.present"; then restore_tmp="${target}.livingip-rollback.$$" sudo rm -f -- "$restore_tmp" || restore_rc=1 if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then @@ -283,15 +644,65 @@ restore_path() { return "$restore_rc" } +validate_restored_path() { + local target="$1" name="$2" + if sudo test -f "$backup_dir/$name.present"; then + sudo test -f "$target" + sudo test ! -L "$target" + sudo cmp -s -- "$backup_dir/$name" "$target" + test "$(sudo stat -c '%u:%g:%a' "$target")" = \ + "$(sudo stat -c '%u:%g:%a' "$backup_dir/$name")" + else + sudo test ! -e "$target" + sudo test ! -L "$target" + fi +} + +validate_restored_directory() { + local target="$1" name="$2" expected_state + if sudo test -f "$backup_dir/$name.present"; then + expected_state="$(sudo cat "$backup_dir/$name.metadata" | tr ' ' ':')" + sudo test -d "$target" + sudo test ! -L "$target" + test "$(sudo stat -c '%u:%g:%a' "$target")" = "$expected_state" + else + sudo test ! -e "$target" + sudo test ! -L "$target" + fi +} + rollback_runtime() { local rollback_rc=0 echo "Deployment failed; restoring the complete previous runtime set." >&2 sudo systemctl stop "$SERVICE" || rollback_rc=1 - restore_path "$profile_bin/teleo-kb" wrapper || rollback_rc=1 - restore_path "$profile_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 + restore_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1 + restore_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 + restore_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 + restore_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 - restore_path "$profile_bin/.livingip-runtime-revision" revision || rollback_rc=1 - sudo systemctl daemon-reload || rollback_rc=1 + restore_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 + restore_directory_state "$runtime_bin" runtime_bin || rollback_rc=1 + restore_directory_state "$runtime_parent" runtime_parent || rollback_rc=1 + restore_directory_state "$runtime_grandparent" runtime_grandparent || rollback_rc=1 + restore_directory_state "$dropin_dir" dropin_dir || rollback_rc=1 + if [ "$rollback_rc" -eq 0 ]; then + validate_restored_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1 + validate_restored_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 + validate_restored_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 + validate_restored_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 + validate_restored_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 + validate_restored_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 + validate_restored_directory "$runtime_grandparent" runtime_grandparent || rollback_rc=1 + validate_restored_directory "$runtime_parent" runtime_parent || rollback_rc=1 + validate_restored_directory "$runtime_bin" runtime_bin || rollback_rc=1 + validate_restored_directory "$dropin_dir" dropin_dir || rollback_rc=1 + fi + if [ "$rollback_rc" -eq 0 ]; then + sudo systemctl daemon-reload || rollback_rc=1 + fi + if [ "$rollback_rc" -eq 0 ]; then + assert_unit_configuration false || rollback_rc=1 + fi if [ "$rollback_rc" -eq 0 ]; then sudo systemctl start "$SERVICE" || rollback_rc=1 assert_service_running || rollback_rc=1 @@ -312,7 +723,13 @@ on_exit() { rc=1 fi fi - sudo rm -f -- "$dropin_stage" || true + sudo rm -f -- \ + "$wrapper_stage" \ + "$tool_stage" \ + "$permission_verifier_stage" \ + "$service_env_verifier_stage" \ + "$revision_stage" \ + "$dropin_stage" || true sudo rm -rf -- "$remote_tmp" || true exit "$rc" } @@ -321,36 +738,117 @@ trap 'on_exit $?' EXIT trap 'exit 130' INT trap 'exit 143' TERM -sudo rm -rf -- "$remote_tmp" -mkdir -p "$backup_dir" -tar -C "$remote_tmp" -xf - +sudo chmod 0711 "$remote_tmp" +sudo install -d -o root -g root -m 0755 "$payload_dir" +sudo install -d -o root -g root -m 0700 "$backup_dir" +sudo install -d -o teleo -g teleo -m 0700 "$work_dir" +sudo tar --no-same-owner --no-same-permissions --no-xattrs --no-acls --no-selinux -C "$payload_dir" -xf - +if sudo find "$payload_dir" -type l -print -quit | grep -q .; then + echo "Staged runtime payload contains a symlink." >&2 + exit 65 +fi +if sudo find "$payload_dir" ! -type d ! -type f -print -quit | grep -q .; then + echo "Staged runtime payload contains a non-regular entry." >&2 + exit 65 +fi +sudo chown -R root:root "$payload_dir" +sudo find "$payload_dir" -type d -exec chmod 0555 {} + +sudo find "$payload_dir" -type f -exec chmod 0444 {} + +sudo chmod 0555 \ + "$payload_dir/$WRAPPER_REL" \ + "$payload_dir/$TOOL_REL" \ + "$payload_dir/$PERMISSION_VERIFIER_REL" \ + "$payload_dir/$SERVICE_ENV_VERIFIER_REL" +test "$(sudo sha256sum "$payload_dir/$WRAPPER_REL" | awk '{print $1}')" = "$WRAPPER_SHA" +test "$(sudo sha256sum "$payload_dir/$TOOL_REL" | awk '{print $1}')" = "$TOOL_SHA" +test "$(sudo sha256sum "$payload_dir/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" +test "$(sudo sha256sum "$payload_dir/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" +test "$(sudo sha256sum "$payload_dir/$DROPIN_REL" | awk '{print $1}')" = "$DROPIN_SHA" +test "$(sudo stat -c '%U:%G:%a' "$remote_tmp")" = "root:root:711" +test "$(sudo stat -c '%U:%G:%a' "$payload_dir")" = "root:root:555" +test "$(sudo stat -c '%U:%G:%a' "$backup_dir")" = "root:root:700" +test "$(sudo stat -c '%U:%G:%a' "$work_dir")" = "teleo:teleo:700" +sudo -n -u teleo test ! -w "$remote_tmp" +sudo -n -u teleo test ! -w "$payload_dir" +sudo -n -u teleo test ! -w "$backup_dir" +sudo test -d "$runtime_anchor" +sudo test -d "$dropin_parent" +sudo test ! -L "$runtime_anchor" +sudo test ! -L "$dropin_parent" +test "$(sudo stat -c '%U:%G' "$runtime_anchor")" = "root:root" +test "$(sudo stat -c '%U:%G' "$dropin_parent")" = "root:root" +sudo -n -u teleo test ! -w "$runtime_anchor" +sudo -n -u teleo test ! -w "$dropin_parent" +for existing_directory in \ + "$runtime_grandparent" \ + "$runtime_parent" \ + "$runtime_bin" \ + "$dropin_dir"; do + assert_existing_root_directory "$existing_directory" +done +for existing_file in \ + "$runtime_bin/teleo-kb" \ + "$runtime_bin/cloudsql_memory_tool.py" \ + "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" \ + "$runtime_bin/verify_gcp_leoclean_service_environment.py" \ + "$runtime_bin/.livingip-runtime-revision" \ + "$dropin_dir/$DROPIN_NAME"; do + assert_existing_root_file "$existing_file" +done assert_service_running -run_scoped_status pre "$remote_tmp/$TOOL_REL" "$remote_tmp/gcloud-pre" +preflight_service_before="$(service_fingerprint)" +assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false +run_scoped_status pre "$payload_dir/$TOOL_REL" +assert_administrator_secret_denied +run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" +assert_service_fingerprint "$preflight_service_before" +if [ "$PREFLIGHT_ONLY" = true ]; then + systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager + exit 0 +fi -backup_path "$profile_bin/teleo-kb" wrapper -backup_path "$profile_bin/cloudsql_memory_tool.py" tool +backup_path "$runtime_bin/teleo-kb" wrapper +backup_path "$runtime_bin/cloudsql_memory_tool.py" tool +backup_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier +backup_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier backup_path "$dropin_dir/$DROPIN_NAME" dropin -backup_path "$profile_bin/.livingip-runtime-revision" revision +backup_path "$runtime_bin/.livingip-runtime-revision" revision +backup_directory_state "$runtime_parent" runtime_parent +backup_directory_state "$runtime_grandparent" runtime_grandparent +backup_directory_state "$runtime_bin" runtime_bin +backup_directory_state "$dropin_dir" dropin_dir mutation_started=true sudo systemctl stop "$SERVICE" -sudo install -d -o teleo -g teleo -m 0755 "$profile_bin" -sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$WRAPPER_REL" "$profile_bin/teleo-kb" -sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$TOOL_REL" "$profile_bin/cloudsql_memory_tool.py" +sudo install -d -o root -g root -m 0755 "$runtime_parent" +sudo install -d -o root -g root -m 0755 "$runtime_bin" +sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage" +sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage" +sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" "$permission_verifier_stage" +sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" "$service_env_verifier_stage" +sudo mv -f -- "$wrapper_stage" "$runtime_bin/teleo-kb" +sudo mv -f -- "$tool_stage" "$runtime_bin/cloudsql_memory_tool.py" +sudo mv -f -- "$permission_verifier_stage" "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" +sudo mv -f -- "$service_env_verifier_stage" "$runtime_bin/verify_gcp_leoclean_service_environment.py" sudo install -d -o root -g root -m 0755 "$dropin_dir" -sudo install -o root -g root -m 0644 "$remote_tmp/$DROPIN_REL" "$dropin_stage" +sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL" "$dropin_stage" sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME" -printf '%s\n' "$SOURCE_REVISION" | sudo tee "$profile_bin/.livingip-runtime-revision" >/dev/null -sudo chown teleo:teleo "$profile_bin/.livingip-runtime-revision" -sudo chmod 0644 "$profile_bin/.livingip-runtime-revision" +printf '%s\n' "$SOURCE_REVISION" | sudo tee "$revision_stage" >/dev/null +sudo chown root:root "$revision_stage" +sudo chmod 0644 "$revision_stage" +sudo mv -f -- "$revision_stage" "$runtime_bin/.livingip-runtime-revision" +assert_installed_files sudo systemctl daemon-reload +assert_unit_configuration sudo systemctl start "$SERVICE" assert_deployed_files assert_service_running -assert_runtime_environment -run_scoped_status post "$profile_bin/cloudsql_memory_tool.py" "$remote_tmp/gcloud-post" +assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py" +run_service_wrapper_status post +assert_administrator_secret_denied +run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" "$work_dir/permission-post/receipt.json" systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager mutation_started=false REMOTE @@ -358,4 +856,8 @@ REMOTE sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}" COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \ - "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command" + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command" diff --git a/docs/gcp-leoclean-runtime-reconciliation.md b/docs/gcp-leoclean-runtime-reconciliation.md index 754b15b..eafcb50 100644 --- a/docs/gcp-leoclean-runtime-reconciliation.md +++ b/docs/gcp-leoclean-runtime-reconciliation.md @@ -2,8 +2,9 @@ ## Scope -This runbook reconciles PR `#145` with the non-production GCP parallel Leo -service and removes its dependency on the PostgreSQL administrator credential. +This runbook reconciles PR `#148`, the least-privilege follow-up to PR `#145`, +with the non-production GCP parallel Leo service and removes its dependency on +the PostgreSQL administrator credential. It does not promote GCP, change the Telegram destination, copy newer VPS data, or make `teleo_canonical` production-authoritative. @@ -31,7 +32,8 @@ IAM or networking. commit. 2. `TELEO_KB_MODE=cloudsql`; missing tools or credentials fail closed. 3. Canonical zero-hit searches do not consult `teleo_restore` unless an - operator explicitly opts in. + operator explicitly opts in. The service is pinned to database + `teleo_canonical` with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`. 4. Leo can read the named canonical tables and stage only a `pending_review` proposal through `kb_stage.stage_leoclean_proposal(...)`. 5. Leo cannot directly insert, update, or delete `public.*` or @@ -55,7 +57,7 @@ required runtime secret has an equivalent secret-level binding. ### 2. Merge reviewed repository code -Run CI on the completed PR `#145` repair before deployment. The deployment +Run CI on the completed PR `#148` repair before deployment. The deployment source must be the resulting merged commit, not a working tree or unmerged branch. @@ -73,6 +75,9 @@ private VM path. Supply the new runtime password through `TELEO_LEOCLEAN_DB_PASSWORD`; the SQL file does not contain it. The migration: - creates or rotates `leoclean_kb_runtime`; +- keeps `leoclean_kb_runtime` `NOLOGIN` while both the `teleo_canonical` grant + phase and the legacy `teleo_kb` denial-verification phase run, and enables + `LOGIN` only in the final transaction after both phases pass; - creates a dedicated `NOLOGIN` function owner with no role memberships; - removes stale table, column, sequence, function, and role-membership grants; - grants exact canonical reads; @@ -94,26 +99,116 @@ by a `pg_catalog, pg_temp` search path, schema-qualified relations, a fixed The administrator password is used only for this bounded bootstrap. Retain no password output. +`teleo_kb` can still be connectable through its existing `PUBLIC CONNECT` +grant. That database-wide default is not treated as runtime authority. The +migration denies the scoped role access to the `teleo_restore` schema, the +runtime client is pinned to `teleo_canonical`, and audit fallback remains +disabled with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`. + ### 5. Preflight and deploy the merged runtime From the exact merged checkout, run: ```bash deploy/sync-gcp-leoclean-runtime.sh --dry-run +deploy/sync-gcp-leoclean-runtime.sh --preflight-only deploy/sync-gcp-leoclean-runtime.sh --restart ``` -The deploy script runs a scoped, redacted database status preflight before it -installs files or restarts the service. It refuses dirty or unmerged runtime -files and refuses a live install without `--restart`. After preflight it stops -the parallel service, atomically replaces the existing Cloud SQL systemd -drop-in, installs the reviewed wrapper and helper, records the Git revision, -starts the service, and verifies all three file hashes. A failure after mutation -restores the prior wrapper, helper, drop-in, and revision as one set before the -old service is restarted. +`--preflight-only` extracts the candidate helper and permission verifier into a +root-created, root-owned, non-writable VM payload directory, verifies the actual +service `MainPID` environment, runs the candidate private Cloud SQL status path, +and runs the sanitized positive-and-negative permission verifier. A separate +`teleo`-owned directory holds only sanitized receipts; it never feeds an install +or rollback. The preflight does not stop or restart the service and does not +install or replace runtime files. It +captures `ActiveState`, `SubState`, `MainPID`, and `NRestarts` before and after +the checks and fails if any value changes. The preflight must pass before +`--restart` is attempted. + +The deploy script refuses dirty, symlinked, or unmerged runtime files and +refuses a live install without `--restart`. Its payload and rollback backup are +root-owned and non-writable by `teleo`; existing runtime intermediates and +targets must be real root-owned, non-writable paths. After preflight it stops +the parallel service, atomically replaces the existing Cloud SQL systemd drop-in, installs the +reviewed wrapper, helper, and permission verifier into the root-controlled +`/usr/local/libexec/livingip/leoclean-kb` directory, records the Git revision, +and verifies every installed hash, owner, mode, path type, and revision before +systemd reloads the unit or starts the service. The wrapper uses fixed +`/bin/bash` and `/usr/bin/python3` interpreters, and every directory on the +service `PATH` must be root-owned and non-writable by `teleo`. Every ancestor of that +directory, the directory itself, the wrapper, helper, verifier, and revision +marker must be non-writable by `teleo`. The systemd service receives that +directory as a read-only bind at its existing +`/home/teleo/.hermes/profiles/leoclean/bin` path; verification compares the +source and service-namespace inode and hashes and proves the mount is read-only. +To prevent the `teleo` service user from renaming a writable ancestor and +recreating that absolute path, the service namespace mounts +`/home/teleo/.hermes` read-only, allows writes back only to the profile's +`state` and `workspace` subdirectories, and removes `CAP_SYS_ADMIN`. Live +verification requires `.hermes` and `bin` to be read-only, both allowed +subdirectories to be writable, `/home` to be non-writable by `teleo`, and the +effective capability set to exclude `CAP_SYS_ADMIN`. + +Preflight, post-restart, and `--verify-only` checks inspect the environment of the actual +systemd `MainPID` through `/proc//environ`, rather than trusting the +configured unit environment alone. A standalone fail-closed verifier requires +the exact reviewed private target and runtime role, requires +`TELEO_GCP_METADATA_ONLY=1`, and rejects all PostgreSQL, Cloud SDK, proxy, TLS +trust, broader Google credential, shell-startup, dynamic-loader, Python-loader, +and administrator-secret overrides without printing their values. Effective +`EnvironmentFile` use and drop-ins ordered after the reviewed configuration are +rejected before restart. This blocks +pre-wrapper injection through fields such as `BASH_ENV`, any `LD_*`, +`PYTHONPATH`, `PYTHONHTTPSVERIFY`, or `SSLKEYLOGFILE`. The checks also require +the reviewed drop-in to appear in `DropInPaths`. The wrapper status probe then +imports the validated environment from the actual `MainPID` with `nsenter +--env`, enters its mount namespace, drops to `teleo`, and invokes the bound +wrapper without reinjecting any database identity setting. + +The service runtime bypasses Cloud SDK configuration and credential caches +entirely. The helper obtains the attached service-account email and a bounded +access token from the fixed GCE metadata endpoint, requires the exact expected +service account, and accesses only the named scoped Secret Manager version over +the fixed HTTPS API. Its URL opener disables proxies. The scoped status read +and administrator-secret IAM denial are rerun after restart. The denial probe +imports the actual systemd `MainPID` environment and mount namespace, drops to +the service's `teleo` identity, and must observe the exact +`secretmanager.versions.access` permission denial. No token or +secret value is retained. The status read executes the bound `teleo-kb` wrapper +from inside the service mount namespace, proving the wrapper and its adjacent +helper rather than bypassing them with a direct helper invocation. A failure +after mutation restores the prior runtime/drop-in directory existence and +metadata, wrapper, helper, both verifiers, drop-in, and revision as one set, +validates it against the root-only backup, and only then reloads systemd and +restarts the old service. ### 6. Verify positive and negative behavior +`--preflight-only`, `--restart`, and `--verify-only` run the verifier as the +`teleo` Unix user. To rerun the installed verifier directly on the VM and +retain its sanitized JSON receipt, use: + +```bash +receipt_dir="$(mktemp -d /tmp/leoclean-permission-receipt.XXXXXX)" +sudo chown teleo:teleo "$receipt_dir" +sudo chmod 0700 "$receipt_dir" +run_id="$(date -u +%Y%m%d%H%M%S)-manual" +sudo -n -u teleo env -i HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + python3 /usr/local/libexec/livingip/leoclean-kb/verify_gcp_leoclean_runtime_permissions.py \ + --run-id "$run_id" \ + --output "$receipt_dir/permission-receipt.json" +sudo stat -c '%U:%G:%a' "$receipt_dir/permission-receipt.json" +``` + +The command must exit zero, print the same canonical JSON that it writes, and +leave the receipt as `teleo:teleo:600`. A passing receipt must report +`status=pass`, `mode=live_private_gcp_staging`, and both `required_tier` and +`current_tier` as `T3_live_readonly`. Those values are a receipt contract, not +a claim that this runbook has already been exercised successfully on the live +staging VM. + Retain a redacted receipt containing: - merged Git commit and deployed file hashes; @@ -122,13 +217,28 @@ Retain a redacted receipt containing: configuration; - `current_database()` and `current_user` showing `teleo_canonical|leoclean_kb_runtime`; +- private server address `10.61.0.3`, port `5432`, and an active SSL session; +- exact safe runtime-role attributes and zero membership edges in + `pg_auth_members`; +- zero effective database/schema `CREATE`, persistent ownership, table/column + DML, sequence privileges, SELECT outside the exact table allowlist, or + executable unexpected `SECURITY DEFINER` functions, plus the exact staging + function owner/search-path/ACL contract; - a real canonical status/search receipt; - a transaction-rolled-back call to `stage_leoclean_proposal(...)` that returns - `pending_review`; -- denied direct insert into `kb_stage.kb_proposals`; -- denied canonical `public.*` write; -- denied reviewer/apply security-definer function; -- denied forged proposer identity and denied `SET ROLE` escalation; + `pending_review` for canonical proposer `leo`, with zero matching canary rows + both before and after the transaction; +- denied direct insert, update, and delete on `kb_stage.kb_proposals`; +- denied insert, update, and delete on canonical `public.claims`; +- denied reviewer/apply security-definer functions; +- exact function-catalog posture: only the five-argument staging function is + executable, the forged six-argument overload is absent, and each expected + reviewer/apply function exists but is not executable by the runtime role; +- unavailable forged-proposer overload and denied `SET ROLE` escalation into + the staging owner, reviewer, apply, or administrator roles; +- denied access to `teleo_restore` even when `teleo_kb` remains connectable; +- readable scoped runtime secret and an IAM permission denial for the + administrator secret, with neither secret value retained; - zero Telegram messages and zero committed canary rows. ### 7. Remove administrator-secret access diff --git a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py index dd2edbe..3367207 100755 --- a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py +++ b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py @@ -1,20 +1,24 @@ #!/usr/bin/env python3 """Cloud SQL memory and KB-proposal bridge for the GCP leoclean profile. -Canonical claim/strategy application remains review-gated. Read commands query -the canonical Teleo KB first, then fall back to the restored `teleo_restore` -shadow schema in Cloud SQL. Proposal commands write staged review records into -`kb_stage.kb_proposals`; they do not apply canonical truth directly. +Canonical claim/strategy application remains review-gated. The production-like +Leo runtime is pinned to the private canonical Cloud SQL database and never +falls back to the restored `teleo_restore` shadow schema. Proposal commands +write staged review records into `kb_stage.kb_proposals`; they do not apply +canonical truth directly. """ from __future__ import annotations import argparse +import base64 +import binascii import hashlib import json import os import re import subprocess +import urllib.request from typing import Any STOPWORDS = { @@ -76,6 +80,38 @@ DEFAULT_CLAIM_BASE_URL = "https://leo.livingip.xyz" RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1" RUNTIME_DB_USER = "leoclean_kb_runtime" RUNTIME_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" +RUNTIME_GCP_PROJECT = "teleo-501523" +RUNTIME_CLOUDSQL_HOST = "10.61.0.3" +RUNTIME_CLOUDSQL_PORT = "5432" +RUNTIME_CLOUDSQL_DB = "teleo_canonical" +RUNTIME_SERVICE_ACCOUNT = "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" +GCE_METADATA_EMAIL_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" +GCE_METADATA_TOKEN_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" +RUNTIME_SECRET_ACCESS_URL = ( + "https://secretmanager.googleapis.com/v1/projects/teleo-501523/secrets/" + "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password/versions/latest:access" +) +METADATA_HEADERS = {"Metadata-Flavor": "Google"} +HTTP_TIMEOUT_SECONDS = 3.0 +MAX_METADATA_EMAIL_BYTES = 256 +MAX_METADATA_TOKEN_BYTES = 4096 +MAX_SECRET_RESPONSE_BYTES = 8192 +MAX_RUNTIME_SECRET_BYTES = 4096 +RUNTIME_TLS_TRUST_ENVIRONMENT = frozenset( + { + "CURL_CA_BUNDLE", + "PYTHONHTTPSVERIFY", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SSLKEYLOGFILE", + } +) +RUNTIME_PSQL_ENV_BASE = { + "PATH": "/usr/bin:/bin", + "PGSSLMODE": "require", + "PGCONNECT_TIMEOUT": "8", +} CLONE_READ_ONLY_PGOPTIONS = "-c default_transaction_read_only=on" RECEIPTED_READ_COMMANDS = frozenset( { @@ -93,6 +129,18 @@ RECEIPTED_READ_COMMANDS = frozenset( ) +class _RejectRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *_args: Any, **_kwargs: Any) -> None: + return None + + +RUNTIME_PROXY_HANDLER = urllib.request.ProxyHandler({}) +RUNTIME_HTTP_OPENER = urllib.request.build_opener( + RUNTIME_PROXY_HANDLER, + _RejectRedirectHandler(), +) + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--host", default=os.environ.get("TELEO_CLOUDSQL_HOST", "10.61.0.3")) @@ -246,8 +294,47 @@ def validate_runtime_connection(args: argparse.Namespace) -> None: raise SystemExit(f"Leo runtime requires database user {RUNTIME_DB_USER}.") if password_secret != RUNTIME_PASSWORD_SECRET: raise SystemExit(f"Leo runtime requires scoped password secret {RUNTIME_PASSWORD_SECRET}.") - if "PGPASSWORD" in os.environ: - raise SystemExit("Refusing inherited PGPASSWORD in Leo runtime mode.") + expected_fields = { + "project": RUNTIME_GCP_PROJECT, + "host": RUNTIME_CLOUDSQL_HOST, + "port": RUNTIME_CLOUDSQL_PORT, + "db": RUNTIME_CLOUDSQL_DB, + "canonical_db": RUNTIME_CLOUDSQL_DB, + } + for field, expected in expected_fields.items(): + actual = str(getattr(args, field, "") or "") + if actual != expected: + raise SystemExit(f"Leo runtime requires {field}={expected}.") + if os.environ.get("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") != "0": + raise SystemExit("Leo runtime requires TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0.") + if os.environ.get("TELEO_GCP_METADATA_ONLY") != "1": + raise SystemExit("Leo runtime requires TELEO_GCP_METADATA_ONLY=1.") + forbidden_environment = any( + normalized_key.startswith("PG") + or normalized_key.startswith("CLOUDSDK_") + or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS" + or normalized_key.startswith("LD_") + or normalized_key.startswith("BASH_FUNC_") + or (normalized_key.startswith("PYTHON") and normalized_key != "PYTHONUNBUFFERED") + or normalized_key in { + "BASH_ENV", + "BASHOPTS", + "CDPATH", + "ENV", + "GLOBIGNORE", + "IFS", + "PS4", + "SHELLOPTS", + } + or normalized_key == "PROXY" + or normalized_key.endswith("_PROXY") + or normalized_key in RUNTIME_TLS_TRUST_ENVIRONMENT + for normalized_key in (key.upper() for key in os.environ) + ) + if forbidden_environment: + raise SystemExit( + "Refusing inherited credential, proxy, TLS, or PostgreSQL environment in Leo runtime mode." + ) return if mode != "clone-readonly": @@ -261,8 +348,8 @@ def validate_runtime_connection(args: argparse.Namespace) -> None: raise SystemExit("Clone credential mode permits read commands only.") if pgoptions != CLONE_READ_ONLY_PGOPTIONS: raise SystemExit("Clone credential mode requires server-enforced default_transaction_read_only=on.") - if not user or (not password_secret and not os.environ.get("PGPASSWORD")): - raise SystemExit("Clone credential mode requires an explicit user and credential source.") + if not user or not os.environ.get("PGPASSWORD"): + raise SystemExit("Clone credential mode requires an explicit user and inherited test credential.") def terms_for(query: str) -> list[str]: @@ -305,48 +392,138 @@ def sql_array(values: list[str], cast: str = "text") -> str: return "array[" + ",".join(sql_literal(value) for value in values) + f"]::{cast}[]" +def _read_http_response(request: urllib.request.Request, *, max_bytes: int, purpose: str) -> bytes: + try: + with RUNTIME_HTTP_OPENER.open(request, timeout=HTTP_TIMEOUT_SECONDS) as response: + if response.getcode() != 200: + raise SystemExit(f"{purpose} request failed.") + body = response.read(max_bytes + 1) + except SystemExit: + raise + # This is a credential-bearing boundary. Never propagate provider, proxy, + # TLS, or test-double exception text because it can contain bearer data. + except Exception: + raise SystemExit(f"{purpose} request failed.") from None + if not isinstance(body, bytes) or len(body) > max_bytes: + raise SystemExit(f"{purpose} response was invalid.") + return body + + +def _strict_json_object(body: bytes, *, purpose: str) -> dict[str, Any]: + def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError("duplicate JSON key") + value[key] = item + return value + + try: + decoded = body.decode("utf-8", errors="strict") + value = json.loads( + decoded, + object_pairs_hook=reject_duplicate_keys, + parse_constant=lambda _constant: (_ for _ in ()).throw(ValueError("non-finite JSON value")), + ) + except (UnicodeDecodeError, ValueError, TypeError): + raise SystemExit(f"{purpose} response was invalid.") from None + if not isinstance(value, dict): + raise SystemExit(f"{purpose} response was invalid.") + return value + + +def _metadata_request(url: str, *, max_bytes: int, purpose: str) -> bytes: + request = urllib.request.Request(url, headers=METADATA_HEADERS, method="GET") + return _read_http_response(request, max_bytes=max_bytes, purpose=purpose) + + +def runtime_password_from_metadata() -> str: + email_body = _metadata_request( + GCE_METADATA_EMAIL_URL, + max_bytes=MAX_METADATA_EMAIL_BYTES, + purpose="Runtime metadata identity", + ) + try: + service_account = email_body.decode("ascii", errors="strict") + except UnicodeDecodeError: + raise SystemExit("Runtime metadata identity response was invalid.") from None + if service_account != RUNTIME_SERVICE_ACCOUNT: + raise SystemExit("Runtime metadata identity did not match the required service account.") + + token_value = _strict_json_object( + _metadata_request( + GCE_METADATA_TOKEN_URL, + max_bytes=MAX_METADATA_TOKEN_BYTES, + purpose="Runtime metadata token", + ), + purpose="Runtime metadata token", + ) + access_token = token_value.get("access_token") + token_type = token_value.get("token_type") + expires_in = token_value.get("expires_in") + if ( + token_type != "Bearer" + or not isinstance(access_token, str) + or not re.fullmatch(r"[!-~]{1,4096}", access_token) + or not isinstance(expires_in, int) + or isinstance(expires_in, bool) + or expires_in <= 0 + ): + raise SystemExit("Runtime metadata token response was invalid.") + + secret_request = urllib.request.Request( + RUNTIME_SECRET_ACCESS_URL, + headers={"Accept": "application/json", "Authorization": f"Bearer {access_token}"}, + method="GET", + ) + secret_value = _strict_json_object( + _read_http_response( + secret_request, + max_bytes=MAX_SECRET_RESPONSE_BYTES, + purpose="Scoped runtime secret access", + ), + purpose="Scoped runtime secret access", + ) + payload = secret_value.get("payload") + encoded_secret = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(encoded_secret, str): + raise SystemExit("Scoped runtime secret response was invalid.") + try: + credential_bytes = base64.b64decode(encoded_secret, validate=True) + except (binascii.Error, ValueError): + raise SystemExit("Scoped runtime secret response was invalid.") from None + if not credential_bytes or len(credential_bytes) > MAX_RUNTIME_SECRET_BYTES: + raise SystemExit("Scoped runtime secret response was invalid.") + if any(byte in credential_bytes for byte in (0, 10, 13)): + raise SystemExit("Scoped runtime secret response was invalid.") + try: + return credential_bytes.decode("utf-8", errors="strict") + except UnicodeDecodeError: + raise SystemExit("Scoped runtime secret response was invalid.") from None + + def password(args: argparse.Namespace) -> str: validate_runtime_connection(args) - if args.credential_mode == "clone-readonly" and os.environ.get("PGPASSWORD"): + if args.credential_mode == "runtime": + return runtime_password_from_metadata() + if os.environ.get("PGPASSWORD"): return os.environ["PGPASSWORD"] - if not args.password_secret: - raise SystemExit("Scoped Cloud SQL password secret is not configured.") - result = subprocess.run( - [ - "gcloud", - "secrets", - "versions", - "access", - "latest", - f"--secret={args.password_secret}", - f"--project={args.project}", - ], - text=True, - capture_output=True, - check=False, - ) - if result.returncode != 0: - raise SystemExit(f"Secret access failed: {result.stderr.strip()}") - credential = result.stdout.strip() - if not credential: - raise SystemExit("Scoped Cloud SQL password secret resolved to an empty value.") - return credential + raise SystemExit("Clone credential mode requires an inherited test credential.") def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: - env = os.environ.copy() - for key in tuple(env): - if key.startswith("PG"): - env.pop(key, None) + credential = password(args) + if args.credential_mode == "runtime": + env = dict(RUNTIME_PSQL_ENV_BASE) + else: + env = {key: value for key, value in os.environ.items() if not key.startswith("PG")} env.update( { "PGHOST": str(args.host), "PGPORT": str(args.port), "PGDATABASE": str(db or args.db), "PGUSER": str(args.user), - "PGSSLMODE": "require", - "PGCONNECT_TIMEOUT": "8", - "PGPASSWORD": password(args), + "PGPASSWORD": credential, } ) if args.credential_mode == "clone-readonly": @@ -360,9 +537,7 @@ def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: check=False, ) if result.returncode != 0: - raise SystemExit( - f"Cloud SQL query failed ({result.returncode})\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" - ) + raise SystemExit(f"Cloud SQL query failed with exit status {result.returncode}.") return result.stdout @@ -385,6 +560,10 @@ def database_read_marker(args: argparse.Namespace) -> dict[str, Any]: select jsonb_build_object( 'database', current_database(), 'database_user', current_user, + 'server_addr', inet_server_addr()::text, + 'server_port', inet_server_port(), + 'ssl', coalesce((select ssl from pg_catalog.pg_stat_ssl where pid = pg_backend_pid()), false), + 'ssl_version', (select version from pg_catalog.pg_stat_ssl where pid = pg_backend_pid()), 'system_identifier', (select system_identifier::text from pg_control_system()), 'wal_lsn', pg_current_wal_lsn()::text )::text; @@ -392,12 +571,34 @@ select jsonb_build_object( rows = psql_json_lines(args, sql, db=args.canonical_db) if len(rows) != 1: raise SystemExit("Canonical database read marker returned an invalid row count.") - return rows[0] + marker = rows[0] + expected_database = str(args.canonical_db or "") + expected_user = str(args.user or "") + if marker.get("database") != expected_database or marker.get("database_user") != expected_user: + raise SystemExit("Canonical database read marker returned an unexpected database identity.") + if getattr(args, "credential_mode", "runtime") == "runtime": + if marker.get("server_addr") != RUNTIME_CLOUDSQL_HOST: + raise SystemExit("Canonical database read marker returned an unexpected server address.") + if str(marker.get("server_port")) != RUNTIME_CLOUDSQL_PORT: + raise SystemExit("Canonical database read marker returned an unexpected server port.") + if marker.get("ssl") is not True: + raise SystemExit("Canonical database read marker did not prove an encrypted connection.") + return marker def _read_marker_stable(before: dict[str, Any], after: dict[str, Any]) -> bool: return all( - before.get(key) == after.get(key) for key in ("database", "database_user", "system_identifier", "wal_lsn") + before.get(key) == after.get(key) + for key in ( + "database", + "database_user", + "server_addr", + "server_port", + "ssl", + "ssl_version", + "system_identifier", + "wal_lsn", + ) ) @@ -470,6 +671,10 @@ def build_retrieval_receipt( "attempts": attempts, "database": before.get("database"), "database_user": before.get("database_user"), + "server_addr": before.get("server_addr"), + "server_port": before.get("server_port"), + "ssl": before.get("ssl"), + "ssl_version": before.get("ssl_version"), "system_identifier": before.get("system_identifier"), "wal_lsn_before": before.get("wal_lsn"), "wal_lsn_after": after.get("wal_lsn"), diff --git a/hermes-agent/leoclean-bin/teleo-kb b/hermes-agent/leoclean-bin/teleo-kb index 369f09b..d5b3e15 100755 --- a/hermes-agent/leoclean-bin/teleo-kb +++ b/hermes-agent/leoclean-bin/teleo-kb @@ -1,9 +1,15 @@ -#!/usr/bin/env bash +#!/bin/bash set -euo pipefail PROFILE_DIR=${HERMES_HOME:-/home/teleo/.hermes/profiles/leoclean} KB_TOOL="$PROFILE_DIR/bin/kb_tool.py" -CLOUDSQL_TOOL="$PROFILE_DIR/bin/cloudsql_memory_tool.py" +SCRIPT_PATH=${BASH_SOURCE[0]} +SCRIPT_DIR=${SCRIPT_PATH%/*} +if [ "$SCRIPT_DIR" = "$SCRIPT_PATH" ]; then + SCRIPT_DIR=. +fi +SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)" +CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py" MODE=${TELEO_KB_MODE:-auto} COMMAND=${1:-} @@ -31,17 +37,22 @@ run_cloudsql() { echo "teleo-kb: Cloud SQL tool is missing or not executable: $CLOUDSQL_TOOL" >&2 exit 69 fi - for dependency in python3 psql gcloud; do - if ! command -v "$dependency" >/dev/null 2>&1; then - echo "teleo-kb: required Cloud SQL dependency is unavailable: $dependency" >&2 - exit 69 - fi - done + if [ ! -x /usr/bin/python3 ]; then + echo "teleo-kb: required Cloud SQL dependency is unavailable: /usr/bin/python3" >&2 + exit 69 + fi + if ! command -v psql >/dev/null 2>&1; then + echo "teleo-kb: required Cloud SQL dependency is unavailable: psql" >&2 + exit 69 + fi extra_args=() while IFS= read -r arg; do [ -n "$arg" ] && extra_args+=("$arg") done < <(cloudsql_args) - exec python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}" + if [ "${#extra_args[@]}" -gt 0 ]; then + exec /usr/bin/python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}" + fi + exec /usr/bin/python3 "$CLOUDSQL_TOOL" "$@" } case "$MODE" in @@ -49,10 +60,10 @@ case "$MODE" in run_cloudsql "$@" ;; local) - exec python3 "$KB_TOOL" --local "$@" + exec /usr/bin/python3 "$KB_TOOL" --local "$@" ;; remote) - exec python3 "$KB_TOOL" "$@" + exec /usr/bin/python3 "$KB_TOOL" "$@" ;; auto) # Canonical commands must fail closed on Cloud SQL errors. Falling through @@ -62,9 +73,9 @@ case "$MODE" in run_cloudsql "$@" fi if command -v docker >/dev/null 2>&1; then - exec python3 "$KB_TOOL" --local "$@" + exec /usr/bin/python3 "$KB_TOOL" --local "$@" fi - exec python3 "$KB_TOOL" "$@" + exec /usr/bin/python3 "$KB_TOOL" "$@" ;; *) echo "Unsupported TELEO_KB_MODE=$MODE; expected auto, cloudsql, local, or remote" >&2 diff --git a/ops/gcp_leoclean_runtime_role.sql b/ops/gcp_leoclean_runtime_role.sql index cd090a3..da884c8 100644 --- a/ops/gcp_leoclean_runtime_role.sql +++ b/ops/gcp_leoclean_runtime_role.sql @@ -1,5 +1,4 @@ \set ON_ERROR_STOP on -\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD -- One-time GCP staging provisioning for Leo's Cloud SQL identity. The password -- is supplied only through the process environment and is never stored here. @@ -8,7 +7,10 @@ -- through one narrowly-scoped SECURITY DEFINER function. It cannot inherit or -- SET ROLE into another principal, forge the proposer identity, write directly -- to canonical/staging tables, or inspect the legacy teleo_restore audit schema. -select 'create role leoclean_kb_runtime login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8' +-- Keep the runtime unable to authenticate until both database phases have +-- completed and verified. Any failure before the final transaction therefore +-- leaves a disabled role instead of a partially provisioned login. +select 'create role leoclean_kb_runtime nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8' where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_runtime') \gexec @@ -16,15 +18,19 @@ select 'create role leoclean_kb_stage_owner nologin nosuperuser nocreatedb nocre where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_stage_owner') \gexec -select format('alter role leoclean_kb_runtime password %L', :'runtime_password') -\gexec - alter role leoclean_kb_runtime - with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; + with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; alter role leoclean_kb_stage_owner with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1 password null; +-- Rotate the credential only after an idempotent rerun has disabled any +-- pre-existing LOGIN role. This ordering has an intentional autocommit +-- boundary: interruption can leave NOLOGIN, never a still-enabled old role. +\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD +select format('alter role leoclean_kb_runtime password %L', :'runtime_password') +\gexec + alter role leoclean_kb_runtime reset all; alter role leoclean_kb_stage_owner reset all; @@ -261,7 +267,7 @@ begin select 1 from pg_catalog.pg_roles role_row where role_row.rolname = 'leoclean_kb_runtime' - and (not role_row.rolcanlogin + and (role_row.rolcanlogin or role_row.rolsuper or role_row.rolcreatedb or role_row.rolcreaterole @@ -270,7 +276,7 @@ begin or role_row.rolbypassrls or role_row.rolconnlimit <> 8) ) then - raise exception 'leoclean_kb_runtime has unsafe role attributes'; + raise exception 'leoclean_kb_runtime has unsafe provisioning role attributes'; end if; if exists ( @@ -818,6 +824,36 @@ commit; \connect teleo_canonical +-- LOGIN is the last state change. If this verification fails, the transaction +-- rolls back and the role remains NOLOGIN while prior least-privilege grants +-- stay inert for direct authentication. +begin; + +alter role leoclean_kb_runtime + with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; + +do $login_verification$ +begin + if exists ( + select 1 + from pg_catalog.pg_roles role_row + where role_row.rolname = 'leoclean_kb_runtime' + and (not role_row.rolcanlogin + or role_row.rolsuper + or role_row.rolcreatedb + or role_row.rolcreaterole + or role_row.rolinherit + or role_row.rolreplication + or role_row.rolbypassrls + or role_row.rolconnlimit <> 8) + ) then + raise exception 'leoclean_kb_runtime final login attributes are unsafe'; + end if; +end +$login_verification$; + +commit; + select jsonb_build_object( 'role', rolname, 'superuser', rolsuper, @@ -835,6 +871,7 @@ select jsonb_build_object( 'can_select_claims', has_table_privilege(rolname, 'public.claims', 'SELECT'), 'can_select_proposals', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'SELECT'), 'can_insert_proposals_directly', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'INSERT'), + 'can_connect_legacy_database', has_database_privilege(rolname, 'teleo_kb', 'CONNECT'), 'can_create_database_objects', has_database_privilege(rolname, 'teleo_canonical', 'CREATE'), 'can_create_temp_objects', has_database_privilege(rolname, 'teleo_canonical', 'TEMP'), 'can_stage_proposal', has_function_privilege( diff --git a/ops/verify_gcp_leoclean_runtime_permissions.py b/ops/verify_gcp_leoclean_runtime_permissions.py new file mode 100644 index 0000000..bdd44dd --- /dev/null +++ b/ops/verify_gcp_leoclean_runtime_permissions.py @@ -0,0 +1,1838 @@ +#!/usr/bin/env python3 +"""Retain a sanitized least-privilege receipt for the GCP Leo runtime role. + +This verifier is intentionally service-independent. It must run on the private +GCP VM as ``teleo`` and connects directly to the private Cloud SQL address. The +only database credential is read from the scoped Secret Manager secret into +process memory, passed to each ``psql`` child through ``PGPASSWORD``, and never +included in an argument, message, receipt, or output file. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pwd +import re +import stat +import subprocess +import sys +import tempfile +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +PROJECT_ID = "teleo-501523" +PRIVATE_CLOUDSQL_HOST = "10.61.0.3" +PRIVATE_CLOUDSQL_PORT = 5432 +CANONICAL_DATABASE = "teleo_canonical" +LEGACY_DATABASE = "teleo_kb" +RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime" +STAGE_OWNER_DATABASE_ROLE = "leoclean_kb_stage_owner" +RUNTIME_UNIX_USER = "teleo" +SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" +ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password" +GCLOUD_BIN = "/usr/bin/gcloud" +PSQL_BIN = "/usr/bin/psql" +COMMAND_TIMEOUT_SECONDS = 30 +CHILD_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +RUN_ID_RE = re.compile(r"[a-z0-9][a-z0-9-]{6,62}[a-z0-9]\Z") +SQLSTATE_RE = re.compile(r"(?m)^(?:ERROR|FATAL):\s+([0-9A-Z]{5}):") +IAM_PERMISSION = "secretmanager.versions.access" +REQUIRED_TIER = "T3_live_readonly" +LOWER_TIER = "T2_runtime" +ARTIFACT_NAME = "gcp_leoclean_runtime_permissions" +STAGE_FUNCTION_SIGNATURE = "kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)" + +# PostgreSQL stores the exact dollar-quoted function body in pg_proc.prosrc. +# A source-parity regression below binds this constant to the provisioning SQL, +# while the live verifier compares the catalog body without retaining it. +EXPECTED_STAGE_FUNCTION_SOURCE = """ +declare + staged kb_stage.kb_proposals%rowtype; + proposer_id uuid; +begin + if session_user <> 'leoclean_kb_runtime' then + raise exception 'stage_leoclean_proposal is restricted to leoclean_kb_runtime' + using errcode = '42501'; + end if; + if p_proposal_type not in ('revise_claim', 'revise_strategy', 'add_edge') then + raise exception 'unsupported Leo proposal type: %', p_proposal_type using errcode = '22023'; + end if; + if nullif(btrim(p_source_ref), '') is null then + raise exception 'source_ref is required' using errcode = '22023'; + end if; + if nullif(btrim(p_rationale), '') is null then + raise exception 'rationale is required' using errcode = '22023'; + end if; + if p_payload is null or jsonb_typeof(p_payload) <> 'object' then + raise exception 'proposal payload must be a JSON object' using errcode = '22023'; + end if; + + select agent.id + into proposer_id + from public.agents as agent + where agent.handle = 'leo'; + + if proposer_id is null then + raise exception 'canonical Leo agent row is required' using errcode = '23503'; + end if; + + insert into kb_stage.kb_proposals ( + proposal_type, + status, + proposed_by_handle, + proposed_by_agent_id, + channel, + source_ref, + rationale, + payload + ) values ( + p_proposal_type, + 'pending_review', + 'leo', + proposer_id, + coalesce(nullif(p_channel, ''), 'telegram'), + p_source_ref, + p_rationale, + p_payload + ) + returning * into staged; + + return to_jsonb(staged); +end +""" +EXPECTED_STAGE_FUNCTION_SOURCE_SHA256 = "89f78d5ad7bc7227a4588baf42f7f77e2162b3c4647a250f7c88716b0debfdea" +MAX_STAGE_FUNCTION_SOURCE_BYTES = 65_536 + +STAGE_FUNCTION_METADATA_EXPECTATIONS: tuple[tuple[str, Any], ...] = ( + ("argument_defaults", 0), + ( + "argument_names", + ["p_proposal_type", "p_channel", "p_source_ref", "p_rationale", "p_payload"], + ), + ("argument_types", ["text", "text", "text", "text", "jsonb"]), + ("kind", "f"), + ("language", "plpgsql"), + ("leakproof", False), + ("owner", STAGE_OWNER_DATABASE_ROLE), + ("parallel", "u"), + ("returns_set", False), + ("result", "jsonb"), + ("security_definer", True), + ("strict", False), + ("volatility", "v"), +) + +STAGE_OWNER_INSERT_COLUMNS: tuple[str, ...] = ( + "proposal_type", + "status", + "proposed_by_handle", + "proposed_by_agent_id", + "channel", + "source_ref", + "rationale", + "payload", +) + +FUNCTION_PRIVILEGE_EXPECTATIONS: tuple[tuple[str, str, bool, bool], ...] = ( + ( + "stage_leoclean_proposal_5_arg", + "kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)", + True, + True, + ), + ( + "stage_leoclean_proposal_6_arg", + "kb_stage.stage_leoclean_proposal(text,text,text,text,text,jsonb)", + False, + False, + ), + ( + "approve_strict_proposal", + "kb_stage.approve_strict_proposal(uuid,text,jsonb,text,text)", + True, + False, + ), + ( + "assert_approved_proposal", + "kb_stage.assert_approved_proposal(uuid,text,jsonb,text,uuid,timestamptz,text)", + True, + False, + ), + ( + "finish_approved_proposal", + "kb_stage.finish_approved_proposal(uuid,text,jsonb,text,uuid,timestamptz,text,text)", + True, + False, + ), +) + +READ_TABLE_ALLOWLIST: tuple[tuple[str, str], ...] = ( + ("public", "agents"), + ("public", "claims"), + ("public", "claim_evidence"), + ("public", "claim_edges"), + ("public", "sources"), + ("public", "personas"), + ("public", "strategies"), + ("public", "beliefs"), + ("public", "blindspots"), + ("public", "agent_roles"), + ("public", "peer_models"), + ("public", "behavioral_rules"), + ("public", "contributor_rules"), + ("public", "reasoning_tools"), + ("public", "governance_gates"), + ("kb_stage", "kb_proposals"), +) + +CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( + "column_dml_grants", + "database_create_grants", + "missing_allowed_table_selects", + "owned_database_objects", + "owner_database_create_grants", + "owner_direct_database_acl_entries", + "owner_missing_agent_select_columns", + "owner_missing_proposal_insert_columns", + "owner_schema_create_grants", + "owner_sequence_grants", + "owner_table_dml_grants", + "owner_unexpected_column_dml_grants", + "owner_unexpected_column_selects", + "owner_unexpected_owned_database_objects", + "owner_unexpected_routine_execute", + "owner_unexpected_schema_usage", + "owner_unexpected_table_selects", + "schema_create_grants", + "sequence_grants", + "table_dml_grants", + "unexpected_column_selects", + "unexpected_direct_database_acl_entries", + "unexpected_routine_execute", + "unexpected_schema_usage", + "unexpected_security_definer_execute", + "unexpected_table_selects", +) + +CATALOG_TRUE_FIELDS: tuple[str, ...] = ( + "canonical_connect", + "owner_grant_contract", + "owner_schema_usage", + "runtime_connect_acl_exact", + "runtime_schema_usage", +) + +LEGACY_CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( + "owner_column_privileges", + "owner_routine_privileges", + "owner_schema_privileges", + "owner_sequence_privileges", + "owner_table_privileges", + "runtime_column_privileges", + "runtime_routine_privileges", + "runtime_schema_privileges", + "runtime_sequence_privileges", + "runtime_table_privileges", +) + + +@dataclass(frozen=True) +class CommandResult: + returncode: int + stdout: str + stderr: str + + +Runner = Callable[[list[str], Mapping[str, str], int, bool], CommandResult] + + +@dataclass(frozen=True) +class NegativeCheck: + name: str + database: str + sql: str + expected_sqlstate: str + + +class VerificationError(RuntimeError): + """A deliberately sanitized, machine-readable verification failure.""" + + def __init__( + self, + code: str, + check: str, + *, + expected_sqlstate: str | None = None, + observed_sqlstate: str | None = None, + ) -> None: + self.code = code + self.check = check + self.expected_sqlstate = expected_sqlstate + self.observed_sqlstate = observed_sqlstate + super().__init__(f"{code}:{check}") + + def as_dict(self) -> dict[str, str]: + payload = {"check": self.check, "code": self.code} + if self.expected_sqlstate is not None: + payload["expected_sqlstate"] = self.expected_sqlstate + if self.observed_sqlstate is not None: + payload["observed_sqlstate"] = self.observed_sqlstate + return payload + + +def subprocess_runner( + command: list[str], + env: Mapping[str, str], + timeout: int, + discard_stdout: bool, +) -> CommandResult: + completed = subprocess.run( + command, + check=False, + text=True, + env=dict(env), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL if discard_stdout else subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + return CommandResult( + returncode=completed.returncode, + stdout="" if discard_stdout else (completed.stdout or ""), + stderr=completed.stderr or "", + ) + + +def validate_run_id(value: str) -> str: + if not RUN_ID_RE.fullmatch(value): + raise ValueError("run id must be 8-64 lowercase letters, digits, or hyphens") + return value + + +def _run_id_arg(value: str) -> str: + try: + return validate_run_id(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(str(exc)) from None + + +def _base_child_environment(_source: Mapping[str, str]) -> dict[str, str]: + return { + "HOME": "/home/teleo", + "LANG": "C", + "LC_ALL": "C", + "PATH": CHILD_PATH, + } + + +def _gcloud_environment(source: Mapping[str, str], config_dir: Path) -> dict[str, str]: + env = _base_child_environment(source) + env.update( + { + "CLOUDSDK_CONFIG": str(config_dir), + "CLOUDSDK_CORE_DISABLE_PROMPTS": "1", + "CLOUDSDK_CORE_PROJECT": PROJECT_ID, + } + ) + return env + + +def _psql_environment(source: Mapping[str, str], password: str) -> dict[str, str]: + env = _base_child_environment(source) + env["PGPASSWORD"] = password + return env + + +def _call( + command: list[str], + *, + env: Mapping[str, str], + runner: Runner, + check: str, + discard_stdout: bool = False, +) -> CommandResult: + try: + return runner(command, env, COMMAND_TIMEOUT_SECONDS, discard_stdout) + except FileNotFoundError: + raise VerificationError("command_not_found", check) from None + except subprocess.TimeoutExpired: + raise VerificationError("command_timed_out", check) from None + except OSError: + raise VerificationError("command_os_error", check) from None + + +def _secret_command(secret_name: str) -> list[str]: + return [ + GCLOUD_BIN, + "secrets", + "versions", + "access", + "latest", + f"--secret={secret_name}", + f"--project={PROJECT_ID}", + "--quiet", + ] + + +def _read_scoped_password(*, env: Mapping[str, str], runner: Runner) -> str: + result = _call( + _secret_command(SCOPED_PASSWORD_SECRET), + env=env, + runner=runner, + check="scoped_secret_access", + ) + if result.returncode != 0: + raise VerificationError("scoped_secret_access_failed", "scoped_secret_access") + + password = result.stdout[:-1] if result.stdout.endswith("\n") else result.stdout + if password.endswith("\r"): + password = password[:-1] + if not password or len(password) > 4096 or any(character in password for character in ("\x00", "\r", "\n")): + raise VerificationError("scoped_secret_value_invalid", "scoped_secret_access") + return password + + +def _assert_administrator_secret_denied(*, env: Mapping[str, str], runner: Runner) -> dict[str, Any]: + result = _call( + _secret_command(ADMINISTRATOR_PASSWORD_SECRET), + env=env, + runner=runner, + check="administrator_secret_access", + discard_stdout=True, + ) + if result.returncode == 0: + raise VerificationError("administrator_secret_unexpectedly_readable", "administrator_secret_access") + + permission_denied = re.search(r"\bPERMISSION_DENIED\b", result.stderr, re.IGNORECASE) is not None + exact_permission = IAM_PERMISSION in result.stderr.lower() + if not permission_denied or not exact_permission: + raise VerificationError("administrator_secret_denial_not_iam_permission", "administrator_secret_access") + return { + "classification": "iam_permission_denied", + "permission": IAM_PERMISSION, + "result": "denied", + "stdout_discarded": True, + } + + +def _conninfo(database: str) -> str: + return ( + f"host={PRIVATE_CLOUDSQL_HOST} port={PRIVATE_CLOUDSQL_PORT} dbname={database} " + f"user={RUNTIME_DATABASE_ROLE} sslmode=require connect_timeout=10 " + "application_name=leoclean_runtime_permission_verifier" + ) + + +def _psql_command(database: str, sql: str) -> list[str]: + return [ + PSQL_BIN, + "--no-psqlrc", + "--no-password", + "--quiet", + "--tuples-only", + "--no-align", + "--set=ON_ERROR_STOP=1", + "--set=VERBOSITY=verbose", + f"--dbname={_conninfo(database)}", + f"--command={sql}", + ] + + +def _single_output_line(result: CommandResult, check: str) -> str: + lines = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if len(lines) != 1: + raise VerificationError("unexpected_psql_output_shape", check) + return lines[0] + + +def _run_psql_success( + *, + check: str, + database: str, + sql: str, + env: Mapping[str, str], + runner: Runner, +) -> str: + result = _call( + _psql_command(database, sql), + env=env, + runner=runner, + check=check, + ) + if result.returncode != 0: + observed = sorted(set(SQLSTATE_RE.findall(result.stderr))) + raise VerificationError( + "psql_check_failed", + check, + observed_sqlstate=",".join(observed) if observed else "missing", + ) + return _single_output_line(result, check) + + +def _parse_json_object(raw: str, check: str) -> dict[str, Any]: + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + raise VerificationError("psql_json_invalid", check) from None + if not isinstance(parsed, dict): + raise VerificationError("psql_json_not_object", check) + return parsed + + +def _parse_zero_count(raw: str, check: str) -> int: + if raw != "0": + raise VerificationError("canary_row_count_nonzero", check) + return 0 + + +def _sql_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def normalize_stage_function_source(source: str) -> str: + normalized = source.replace("\r\n", "\n").replace("\r", "\n") + return normalized.strip("\n") + "\n" + + +def _identity_sql() -> str: + return """ +select pg_catalog.jsonb_build_object( + 'database', pg_catalog.current_database(), + 'current_user', current_user, + 'session_user', session_user, + 'server_addr', pg_catalog.inet_server_addr()::text, + 'server_port', pg_catalog.inet_server_port(), + 'ssl', coalesce( + (select ssl from pg_catalog.pg_stat_ssl where pid = pg_catalog.pg_backend_pid()), + false + ), + 'ssl_version', coalesce( + (select version from pg_catalog.pg_stat_ssl where pid = pg_catalog.pg_backend_pid()), + '' + ) +)::text; +""".strip() + + +def _database_role_posture_sql(role_name: str) -> str: + return f""" +select pg_catalog.jsonb_build_object( + 'role', role_row.rolname, + 'can_login', role_row.rolcanlogin, + 'is_superuser', role_row.rolsuper, + 'can_create_db', role_row.rolcreatedb, + 'can_create_role', role_row.rolcreaterole, + 'inherits_privileges', role_row.rolinherit, + 'can_replicate', role_row.rolreplication, + 'bypasses_rls', role_row.rolbypassrls, + 'connection_limit', role_row.rolconnlimit, + 'direct_membership_edges', ( + select pg_catalog.count(*) + from pg_catalog.pg_auth_members membership + where membership.roleid = role_row.oid + or membership.member = role_row.oid + ) +)::text +from pg_catalog.pg_roles role_row +where role_row.rolname = {_sql_literal(role_name)}; +""".strip() + + +def _role_posture_sql() -> str: + return _database_role_posture_sql(RUNTIME_DATABASE_ROLE) + + +def _stage_owner_role_posture_sql() -> str: + return _database_role_posture_sql(STAGE_OWNER_DATABASE_ROLE) + + +def _function_privilege_posture_sql() -> str: + values_sql = ",\n ".join( + f"({_sql_literal(name)}, {_sql_literal(signature)})" + for name, signature, _expected_exists, _expected_execute in FUNCTION_PRIVILEGE_EXPECTATIONS + ) + return f""" +with expected(check_name, signature) as ( + values + {values_sql} +), resolved as ( + select check_name, + signature, + pg_catalog.to_regprocedure(signature) as function_oid + from expected +) +select pg_catalog.jsonb_object_agg( + check_name, + pg_catalog.jsonb_build_object( + 'exists', function_oid is not null, + 'execute', coalesce( + pg_catalog.has_function_privilege(current_user, function_oid::pg_catalog.oid, 'EXECUTE'), + false + ) + ) +)::text +from resolved; +""".strip() + + +def _stage_function_definition_sql() -> str: + return f""" +with target as ( + select pg_catalog.to_regprocedure({_sql_literal(STAGE_FUNCTION_SIGNATURE)})::pg_catalog.oid as oid +), function_row as ( + select target.oid, + function_catalog.proowner, + function_catalog.prolang, + function_catalog.prokind, + function_catalog.prosecdef, + function_catalog.proleakproof, + function_catalog.proisstrict, + function_catalog.proretset, + function_catalog.provolatile, + function_catalog.proparallel, + function_catalog.pronargdefaults, + function_catalog.proargnames, + function_catalog.proargmodes, + function_catalog.proargtypes, + function_catalog.proconfig, + function_catalog.proacl, + function_catalog.prosrc, + owner_role.rolname as owner_name, + language_row.lanname as language_name + from target + left join pg_catalog.pg_proc function_catalog on function_catalog.oid = target.oid + left join pg_catalog.pg_roles owner_role on owner_role.oid = function_catalog.proowner + left join pg_catalog.pg_language language_row on language_row.oid = function_catalog.prolang +), runtime_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(RUNTIME_DATABASE_ROLE)} +), owner_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(STAGE_OWNER_DATABASE_ROLE)} +) +select pg_catalog.jsonb_build_object( + 'exists', function_row.oid is not null, + 'owner', function_row.owner_name, + 'language', function_row.language_name, + 'kind', function_row.prokind, + 'security_definer', function_row.prosecdef, + 'leakproof', function_row.proleakproof, + 'strict', function_row.proisstrict, + 'returns_set', function_row.proretset, + 'volatility', function_row.provolatile, + 'parallel', function_row.proparallel, + 'argument_defaults', function_row.pronargdefaults, + 'argument_names', function_row.proargnames, + 'argument_modes', function_row.proargmodes, + 'argument_types', ( + select pg_catalog.jsonb_agg(pg_catalog.format_type(argument_type, null) order by ordinal) + from pg_catalog.unnest(function_row.proargtypes::pg_catalog.oid[]) + with ordinality argument(argument_type, ordinal) + ), + 'result', pg_catalog.pg_get_function_result(function_row.oid), + 'configuration', function_row.proconfig, + 'runtime_execute', coalesce( + pg_catalog.has_function_privilege(current_user, function_row.oid, 'EXECUTE'), + false + ), + 'acl_exact', coalesce(( + select pg_catalog.count(*) filter ( + where acl.grantee = runtime_role.oid + and acl.privilege_type = 'EXECUTE' + and not acl.is_grantable + ) = 1 + and pg_catalog.count(*) filter ( + where not ( + acl.grantee in (runtime_role.oid, owner_role.oid) + and acl.privilege_type = 'EXECUTE' + and (acl.grantee = owner_role.oid or not acl.is_grantable) + ) + ) = 0 + from runtime_role + cross join owner_role + cross join lateral pg_catalog.aclexplode( + coalesce(function_row.proacl, pg_catalog.acldefault('f', function_row.proowner)) + ) acl + ), false), + 'source', function_row.prosrc +)::text +from function_row; +""".strip() + + +def _catalog_privilege_posture_sql() -> str: + allowlist_values = ",\n ".join( + f"({_sql_literal(schema)}, {_sql_literal(relation)})" for schema, relation in READ_TABLE_ALLOWLIST + ) + owner_insert_values = ",\n ".join( + f"({_sql_literal(column_name)})" for column_name in STAGE_OWNER_INSERT_COLUMNS + ) + return f""" +with runtime_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(RUNTIME_DATABASE_ROLE)} +), owner_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(STAGE_OWNER_DATABASE_ROLE)} +), stage_function as ( + select pg_catalog.to_regprocedure({_sql_literal(STAGE_FUNCTION_SIGNATURE)})::pg_catalog.oid as oid +), app_schemas as ( + select oid, nspname + from pg_catalog.pg_namespace + where nspname <> 'information_schema' + and nspname !~ '^pg_' +), allowed_select(nspname, relname) as ( + values + {allowlist_values} +), allowed_relation as ( + select allowed_select.nspname, + allowed_select.relname, + relation.oid + from allowed_select + left join pg_catalog.pg_namespace namespace + on namespace.nspname = allowed_select.nspname + left join pg_catalog.pg_class relation + on relation.relnamespace = namespace.oid + and relation.relname = allowed_select.relname + and relation.relkind in ('r', 'p', 'v', 'm', 'f') +), owner_insert_column(attname) as ( + values + {owner_insert_values} +) +select pg_catalog.jsonb_build_object( + 'canonical_connect', coalesce( + pg_catalog.has_database_privilege(current_user, pg_catalog.current_database(), 'CONNECT'), + false + ), + 'runtime_connect_acl_exact', coalesce(( + select pg_catalog.count(*) filter ( + where database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + and acl.privilege_type = 'CONNECT' + and not acl.is_grantable + ) = 1 + from pg_catalog.pg_database database_row + cross join runtime_role + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) + and acl.grantee = runtime_role.oid + ), false), + 'unexpected_direct_database_acl_entries', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + cross join runtime_role + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) + and acl.grantee = runtime_role.oid + and not ( + database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + and acl.privilege_type = 'CONNECT' + and not acl.is_grantable + ) + ), + 'owner_direct_database_acl_entries', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + cross join owner_role + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) + and acl.grantee = owner_role.oid + ), + 'database_create_grants', ( + select pg_catalog.count(*)::int + from (values ({_sql_literal(CANONICAL_DATABASE)}), ({_sql_literal(LEGACY_DATABASE)})) database_name(name) + where pg_catalog.has_database_privilege(current_user, database_name.name, 'CREATE') + ), + 'owner_database_create_grants', ( + select pg_catalog.count(*)::int + from (values ({_sql_literal(CANONICAL_DATABASE)}), ({_sql_literal(LEGACY_DATABASE)})) database_name(name) + where pg_catalog.has_database_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + database_name.name, + 'CREATE' + ) + ), + 'owned_database_objects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_shdepend dependency + join runtime_role on runtime_role.oid = dependency.refobjid + where dependency.refclassid = 'pg_catalog.pg_authid'::pg_catalog.regclass + and dependency.deptype = 'o' + ), + 'owner_unexpected_owned_database_objects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_shdepend dependency + join owner_role on owner_role.oid = dependency.refobjid + cross join stage_function + where dependency.refclassid = 'pg_catalog.pg_authid'::pg_catalog.regclass + and dependency.deptype = 'o' + and not ( + dependency.dbid = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = pg_catalog.current_database() + ) + and dependency.classid = 'pg_catalog.pg_proc'::pg_catalog.regclass + and dependency.objid = stage_function.oid + and dependency.objsubid = 0 + ) + ), + 'runtime_schema_usage', coalesce( + pg_catalog.has_schema_privilege(current_user, 'public', 'USAGE') + and pg_catalog.has_schema_privilege(current_user, 'kb_stage', 'USAGE'), + false + ), + 'owner_schema_usage', coalesce( + pg_catalog.has_schema_privilege({_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, 'public', 'USAGE') + and pg_catalog.has_schema_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + 'kb_stage', + 'USAGE' + ), + false + ), + 'unexpected_schema_usage', ( + select pg_catalog.count(*)::int + from app_schemas + where app_schemas.nspname not in ('public', 'kb_stage') + and pg_catalog.has_schema_privilege(current_user, app_schemas.oid, 'USAGE') + ), + 'owner_unexpected_schema_usage', ( + select pg_catalog.count(*)::int + from app_schemas + where app_schemas.nspname not in ('public', 'kb_stage') + and pg_catalog.has_schema_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + app_schemas.oid, + 'USAGE' + ) + ), + 'schema_create_grants', ( + select pg_catalog.count(*)::int + from app_schemas + where pg_catalog.has_schema_privilege(current_user, app_schemas.oid, 'CREATE') + ), + 'owner_schema_create_grants', ( + select pg_catalog.count(*)::int + from app_schemas + where pg_catalog.has_schema_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + app_schemas.oid, + 'CREATE' + ) + ), + 'missing_allowed_table_selects', ( + select pg_catalog.count(*)::int + from allowed_relation + where allowed_relation.oid is null + or not coalesce( + pg_catalog.has_table_privilege(current_user, allowed_relation.oid, 'SELECT'), + false + ) + ), + 'table_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege(current_user, relation.oid, privilege.name) + ), + 'column_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege(current_user, relation.oid, attribute.attnum, privilege.name) + ), + 'unexpected_table_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + left join allowed_select + on allowed_select.nspname = namespace.nspname + and allowed_select.relname = relation.relname + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege(current_user, relation.oid, 'SELECT') + and allowed_select.relname is null + ), + 'unexpected_column_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + left join allowed_select + on allowed_select.nspname = namespace.nspname + and allowed_select.relname = relation.relname + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege(current_user, relation.oid, attribute.attnum, 'SELECT') + and allowed_select.relname is null + ), + 'sequence_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where relation.relkind = 'S' + and pg_catalog.has_sequence_privilege(current_user, relation.oid, privilege.name) + ), + 'unexpected_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join stage_function + where function_row.oid is distinct from stage_function.oid + and pg_catalog.has_function_privilege(current_user, function_row.oid, 'EXECUTE') + ), + 'unexpected_security_definer_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join stage_function + where function_row.prosecdef + and function_row.oid is distinct from stage_function.oid + and pg_catalog.has_function_privilege(current_user, function_row.oid, 'EXECUTE') + ), + 'owner_grant_contract', coalesce( + not pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + pg_catalog.to_regclass('public.agents'), + 'SELECT' + ) + and pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + pg_catalog.to_regclass('kb_stage.kb_proposals'), + 'SELECT' + ) + and not pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + pg_catalog.to_regclass('kb_stage.kb_proposals'), + 'INSERT' + ), + false + ), + 'owner_missing_agent_select_columns', ( + select pg_catalog.count(*)::int + from (values ('id'), ('handle')) required_column(attname) + left join pg_catalog.pg_attribute attribute + on attribute.attrelid = pg_catalog.to_regclass('public.agents') + and attribute.attname = required_column.attname + and attribute.attnum > 0 + and not attribute.attisdropped + where attribute.attnum is null + or not coalesce( + pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + attribute.attrelid, + attribute.attnum, + 'SELECT' + ), + false + ) + ), + 'owner_missing_proposal_insert_columns', ( + select pg_catalog.count(*)::int + from owner_insert_column + left join pg_catalog.pg_attribute attribute + on attribute.attrelid = pg_catalog.to_regclass('kb_stage.kb_proposals') + and attribute.attname = owner_insert_column.attname + and attribute.attnum > 0 + and not attribute.attisdropped + where attribute.attnum is null + or not coalesce( + pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + attribute.attrelid, + attribute.attnum, + 'INSERT' + ), + false + ) + ), + 'owner_table_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + privilege.name + ) + ), + 'owner_unexpected_column_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + attribute.attnum, + privilege.name + ) + and not ( + privilege.name = 'INSERT' + and namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' + and attribute.attname in (select attname from owner_insert_column) + ) + ), + 'owner_unexpected_table_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + 'SELECT' + ) + and not (namespace.nspname = 'kb_stage' and relation.relname = 'kb_proposals') + ), + 'owner_unexpected_column_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + attribute.attnum, + 'SELECT' + ) + and not ( + (namespace.nspname = 'kb_stage' and relation.relname = 'kb_proposals') + or ( + namespace.nspname = 'public' + and relation.relname = 'agents' + and attribute.attname in ('id', 'handle') + ) + ) + ), + 'owner_sequence_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where relation.relkind = 'S' + and pg_catalog.has_sequence_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + privilege.name + ) + ), + 'owner_unexpected_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join stage_function + where function_row.oid is distinct from stage_function.oid + and pg_catalog.has_function_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + function_row.oid, + 'EXECUTE' + ) + ) +)::text; +""".strip() + + +def _allowed_read_sql() -> str: + return """ +select pg_catalog.jsonb_build_object( + 'claims_rows_sampled', (select pg_catalog.count(*) from (select 1 from public.claims limit 1) sampled), + 'proposal_rows_sampled', ( + select pg_catalog.count(*) from (select 1 from kb_stage.kb_proposals limit 1) sampled + ) +)::text; +""".strip() + + +def _legacy_catalog_privilege_posture_sql() -> str: + return f""" +with scoped_role(role_name, field_prefix) as ( + values + ({_sql_literal(RUNTIME_DATABASE_ROLE)}, 'runtime'), + ({_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, 'owner') +), restore_schema as ( + select oid + from pg_catalog.pg_namespace + where nspname = 'teleo_restore' +), posture as ( + select scoped_role.field_prefix, + ( + select pg_catalog.count(*)::int + from restore_schema + cross join (values ('USAGE'), ('CREATE')) privilege(name) + where pg_catalog.has_schema_privilege( + scoped_role.role_name, + restore_schema.oid, + privilege.name + ) + ) as schema_privileges, + ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join restore_schema on restore_schema.oid = relation.relnamespace + cross join ( + values ('SELECT'), ('INSERT'), ('UPDATE'), ('DELETE'), + ('TRUNCATE'), ('REFERENCES'), ('TRIGGER') + ) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege( + scoped_role.role_name, + relation.oid, + privilege.name + ) + ) as table_privileges, + ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join restore_schema on restore_schema.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('SELECT'), ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege( + scoped_role.role_name, + relation.oid, + attribute.attnum, + privilege.name + ) + ) as column_privileges, + ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join restore_schema on restore_schema.oid = relation.relnamespace + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where relation.relkind = 'S' + and pg_catalog.has_sequence_privilege( + scoped_role.role_name, + relation.oid, + privilege.name + ) + ) as sequence_privileges, + ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join restore_schema on restore_schema.oid = function_row.pronamespace + where pg_catalog.has_function_privilege( + scoped_role.role_name, + function_row.oid, + 'EXECUTE' + ) + ) as routine_privileges + from scoped_role +) +select pg_catalog.jsonb_build_object( + 'runtime_schema_privileges', max(schema_privileges) filter (where field_prefix = 'runtime'), + 'runtime_table_privileges', max(table_privileges) filter (where field_prefix = 'runtime'), + 'runtime_column_privileges', max(column_privileges) filter (where field_prefix = 'runtime'), + 'runtime_sequence_privileges', max(sequence_privileges) filter (where field_prefix = 'runtime'), + 'runtime_routine_privileges', max(routine_privileges) filter (where field_prefix = 'runtime'), + 'owner_schema_privileges', max(schema_privileges) filter (where field_prefix = 'owner'), + 'owner_table_privileges', max(table_privileges) filter (where field_prefix = 'owner'), + 'owner_column_privileges', max(column_privileges) filter (where field_prefix = 'owner'), + 'owner_sequence_privileges', max(sequence_privileges) filter (where field_prefix = 'owner'), + 'owner_routine_privileges', max(routine_privileges) filter (where field_prefix = 'owner') +)::text +from posture; +""".strip() + + +def _canary_count_sql(source_ref: str) -> str: + return f"select pg_catalog.count(*)::text from kb_stage.kb_proposals where source_ref = {_sql_literal(source_ref)};" + + +def _stage_sql(run_id: str, source_ref: str) -> str: + return f""" +begin; +with staged as ( + select kb_stage.stage_leoclean_proposal( + 'revise_claim', + 'runtime-permission-verification', + {_sql_literal(source_ref)}, + 'least-privilege transaction rollback verification', + pg_catalog.jsonb_build_object('run_id', {_sql_literal(run_id)}, 'verification', true) + ) as proposal +) +select pg_catalog.jsonb_build_object( + 'agent_id_matches', coalesce( + proposal ->> 'proposed_by_agent_id' = ( + select id::text from public.agents where handle = 'leo' + ), + false + ), + 'proposal', proposal +)::text +from staged; +rollback; +""".strip() + + +def _transactional(sql: str) -> str: + return f"begin;\n{sql.rstrip(';')};\nrollback;" + + +def _negative_checks(run_id: str, source_ref: str) -> tuple[NegativeCheck, ...]: + nil_uuid = "00000000-0000-0000-0000-000000000000" + return ( + NegativeCheck( + "proposal_insert", + CANONICAL_DATABASE, + _transactional("insert into kb_stage.kb_proposals select * from kb_stage.kb_proposals where false"), + "42501", + ), + NegativeCheck( + "proposal_update", + CANONICAL_DATABASE, + _transactional("update kb_stage.kb_proposals set status = status where false"), + "42501", + ), + NegativeCheck( + "proposal_delete", + CANONICAL_DATABASE, + _transactional("delete from kb_stage.kb_proposals where false"), + "42501", + ), + NegativeCheck( + "canonical_insert", + CANONICAL_DATABASE, + _transactional("insert into public.claims select * from public.claims where false"), + "42501", + ), + NegativeCheck( + "canonical_update", + CANONICAL_DATABASE, + _transactional("update public.claims set id = id where false"), + "42501", + ), + NegativeCheck( + "canonical_delete", + CANONICAL_DATABASE, + _transactional("delete from public.claims where false"), + "42501", + ), + NegativeCheck( + "forged_proposer_overload", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.stage_leoclean_proposal(" + f"'revise_claim', 'forged-principal', 'verification', {_sql_literal(source_ref)}, " + f"'must be unavailable', pg_catalog.jsonb_build_object('run_id', {_sql_literal(run_id)}))" + ), + "42883", + ), + NegativeCheck( + "reviewer_approve_function", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.approve_strict_proposal(" + f"'{nil_uuid}'::uuid, 'revise_claim', '{{}}'::jsonb, 'leo', 'must be denied')" + ), + "42501", + ), + NegativeCheck( + "apply_assert_function", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.assert_approved_proposal(" + f"'{nil_uuid}'::uuid, 'revise_claim', '{{}}'::jsonb, 'hash', " + f"'{nil_uuid}'::uuid, '2000-01-01T00:00:00Z'::timestamptz, 'leo')" + ), + "42501", + ), + NegativeCheck( + "apply_finish_function", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.finish_approved_proposal(" + f"'{nil_uuid}'::uuid, 'revise_claim', '{{}}'::jsonb, 'hash', " + f"'{nil_uuid}'::uuid, '2000-01-01T00:00:00Z'::timestamptz, 'leo', 'leo')" + ), + "42501", + ), + NegativeCheck( + "set_role_stage_owner", + CANONICAL_DATABASE, + _transactional("set role leoclean_kb_stage_owner"), + "42501", + ), + NegativeCheck( + "set_role_reviewer", + CANONICAL_DATABASE, + _transactional("set role kb_review"), + "42501", + ), + NegativeCheck( + "set_role_apply", + CANONICAL_DATABASE, + _transactional("set role kb_apply"), + "42501", + ), + NegativeCheck( + "set_role_administrator", + CANONICAL_DATABASE, + _transactional("set role postgres"), + "42501", + ), + NegativeCheck( + "legacy_teleo_restore_read", + LEGACY_DATABASE, + _transactional("select 1 from teleo_restore.response_audit limit 1"), + "42501", + ), + ) + + +def _expect_sqlstate( + check: NegativeCheck, + *, + env: Mapping[str, str], + runner: Runner, +) -> dict[str, str]: + result = _call( + _psql_command(check.database, check.sql), + env=env, + runner=runner, + check=check.name, + ) + if result.returncode == 0: + raise VerificationError( + "negative_permission_unexpectedly_succeeded", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate="success", + ) + + observed = SQLSTATE_RE.findall(result.stderr) + unique_observed = sorted(set(observed)) + if not observed: + raise VerificationError( + "negative_permission_sqlstate_missing", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate="missing", + ) + if unique_observed != [check.expected_sqlstate]: + raise VerificationError( + "negative_permission_sqlstate_mismatch", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate=",".join(unique_observed), + ) + return { + "check": check.name, + "expected_sqlstate": check.expected_sqlstate, + "observed_sqlstate": check.expected_sqlstate, + "result": "denied", + } + + +def _assert_identity(identity: dict[str, Any]) -> dict[str, Any]: + expected = { + "database": CANONICAL_DATABASE, + "current_user": RUNTIME_DATABASE_ROLE, + "session_user": RUNTIME_DATABASE_ROLE, + "server_addr": PRIVATE_CLOUDSQL_HOST, + "server_port": PRIVATE_CLOUDSQL_PORT, + "ssl": True, + } + if any(identity.get(key) != value for key, value in expected.items()): + raise VerificationError("private_ssl_identity_mismatch", "database_identity") + ssl_version = identity.get("ssl_version") + if not isinstance(ssl_version, str) or not ssl_version: + raise VerificationError("ssl_version_missing", "database_identity") + return {**expected, "ssl_version": ssl_version} + + +def _assert_role_posture(posture: dict[str, Any]) -> dict[str, Any]: + expected_booleans = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": True, + "can_replicate": False, + "inherits_privileges": False, + "is_superuser": False, + } + if posture.get("role") != RUNTIME_DATABASE_ROLE: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + for key, expected in expected_booleans.items(): + if posture.get(key) is not expected: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + + connection_limit = posture.get("connection_limit") + membership_edges = posture.get("direct_membership_edges") + if isinstance(connection_limit, bool) or not isinstance(connection_limit, int) or connection_limit != 8: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + if isinstance(membership_edges, bool) or not isinstance(membership_edges, int) or membership_edges != 0: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + + return { + "role": RUNTIME_DATABASE_ROLE, + **expected_booleans, + "connection_limit": connection_limit, + "direct_membership_edges": membership_edges, + } + + +def _assert_stage_owner_role_posture(posture: dict[str, Any]) -> dict[str, Any]: + expected_booleans = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": False, + "can_replicate": False, + "inherits_privileges": False, + "is_superuser": False, + } + if posture.get("role") != STAGE_OWNER_DATABASE_ROLE: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + for key, expected in expected_booleans.items(): + if posture.get(key) is not expected: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + + connection_limit = posture.get("connection_limit") + membership_edges = posture.get("direct_membership_edges") + if isinstance(connection_limit, bool) or not isinstance(connection_limit, int) or connection_limit != -1: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + if isinstance(membership_edges, bool) or not isinstance(membership_edges, int) or membership_edges != 0: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + + return { + "role": STAGE_OWNER_DATABASE_ROLE, + **expected_booleans, + "connection_limit": connection_limit, + "direct_membership_edges": membership_edges, + } + + +def _assert_function_privilege_posture(posture: dict[str, Any]) -> dict[str, dict[str, Any]]: + sanitized: dict[str, dict[str, Any]] = {} + for name, signature, expected_exists, expected_execute in FUNCTION_PRIVILEGE_EXPECTATIONS: + observed = posture.get(name) + if not isinstance(observed, dict): + raise VerificationError("function_privilege_posture_mismatch", "function_privilege_posture") + exists = observed.get("exists") + execute = observed.get("execute") + if exists is not expected_exists or execute is not expected_execute: + raise VerificationError("function_privilege_posture_mismatch", "function_privilege_posture") + sanitized[name] = { + "execute": expected_execute, + "exists": expected_exists, + "signature": signature, + } + return sanitized + + +def _assert_stage_function_definition(posture: dict[str, Any]) -> dict[str, Any]: + if posture.get("exists") is not True: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + for field, expected in STAGE_FUNCTION_METADATA_EXPECTATIONS: + observed = posture.get(field) + if type(observed) is not type(expected) or observed != expected: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + if posture.get("argument_modes") is not None: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + if posture.get("configuration") != ["search_path=pg_catalog, pg_temp"]: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + for field in ("acl_exact", "runtime_execute"): + if posture.get(field) is not True: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + + source = posture.get("source") + if not isinstance(source, str) or len(source.encode("utf-8")) > MAX_STAGE_FUNCTION_SOURCE_BYTES: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + source_sha256 = hashlib.sha256(normalize_stage_function_source(source).encode("utf-8")).hexdigest() + if source_sha256 != EXPECTED_STAGE_FUNCTION_SOURCE_SHA256: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + + return { + "acl_exact": True, + "argument_modes": None, + "configuration": ["search_path=pg_catalog, pg_temp"], + "exists": True, + **{field: expected for field, expected in STAGE_FUNCTION_METADATA_EXPECTATIONS}, + "runtime_execute": True, + "signature": STAGE_FUNCTION_SIGNATURE, + "source_sha256": source_sha256, + } + + +def _assert_catalog_privilege_posture(posture: dict[str, Any]) -> dict[str, int | bool]: + sanitized: dict[str, int | bool] = {} + for field in CATALOG_ZERO_COUNT_FIELDS: + value = posture.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value != 0: + raise VerificationError("catalog_privilege_posture_mismatch", "catalog_privilege_posture") + sanitized[field] = 0 + for field in CATALOG_TRUE_FIELDS: + if posture.get(field) is not True: + raise VerificationError("catalog_privilege_posture_mismatch", "catalog_privilege_posture") + sanitized[field] = True + return sanitized + + +def _assert_legacy_catalog_privilege_posture(posture: dict[str, Any]) -> dict[str, int]: + sanitized: dict[str, int] = {} + for field in LEGACY_CATALOG_ZERO_COUNT_FIELDS: + value = posture.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value != 0: + raise VerificationError("legacy_catalog_privilege_posture_mismatch", "legacy_catalog_privileges") + sanitized[field] = 0 + return sanitized + + +def _assert_allowed_reads(readback: dict[str, Any]) -> dict[str, int]: + sanitized: dict[str, int] = {} + for key in ("claims_rows_sampled", "proposal_rows_sampled"): + value = readback.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value not in (0, 1): + raise VerificationError("allowed_read_invalid", "canonical_allowed_reads") + sanitized[key] = value + return sanitized + + +def _assert_stage_result(staged: dict[str, Any], source_ref: str) -> dict[str, Any]: + proposal = staged.get("proposal") + if not isinstance(proposal, dict) or staged.get("agent_id_matches") is not True: + raise VerificationError("staged_proposal_agent_mismatch", "rolled_back_proposal_stage") + expected = { + "proposed_by_handle": "leo", + "source_ref": source_ref, + "status": "pending_review", + } + if any(proposal.get(key) != value for key, value in expected.items()): + raise VerificationError("staged_proposal_contract_mismatch", "rolled_back_proposal_stage") + proposed_by_agent_id = proposal.get("proposed_by_agent_id") + if not isinstance(proposed_by_agent_id, str) or not proposed_by_agent_id: + raise VerificationError("staged_proposal_agent_missing", "rolled_back_proposal_stage") + return {**expected, "agent_id_matches": True, "proposed_by_agent_id": proposed_by_agent_id} + + +def verify_runtime_permissions( + run_id: str, + *, + runner: Runner = subprocess_runner, + base_env: Mapping[str, str] | None = None, + effective_user: str | None = None, +) -> dict[str, Any]: + try: + run_id = validate_run_id(run_id) + except ValueError: + raise VerificationError("run_id_invalid", "arguments") from None + + if effective_user is None: + try: + effective_user = pwd.getpwuid(os.geteuid()).pw_name + except KeyError: + raise VerificationError("unix_user_unknown", "runtime_user") from None + if effective_user != RUNTIME_UNIX_USER: + raise VerificationError("unexpected_unix_user", "runtime_user") + + source = os.environ if base_env is None else base_env + source_ref = f"leo-runtime-permission:{run_id}" + generated_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + with tempfile.TemporaryDirectory(prefix="leoclean-runtime-permission-gcloud-") as config_name: + config_dir = Path(config_name) + config_dir.chmod(0o700) + gcloud_env = _gcloud_environment(source, config_dir) + password = _read_scoped_password(env=gcloud_env, runner=runner) + psql_env = _psql_environment(source, password) + + identity = _assert_identity( + _parse_json_object( + _run_psql_success( + check="database_identity", + database=CANONICAL_DATABASE, + sql=_identity_sql(), + env=psql_env, + runner=runner, + ), + "database_identity", + ) + ) + role_posture = _assert_role_posture( + _parse_json_object( + _run_psql_success( + check="runtime_role_posture", + database=CANONICAL_DATABASE, + sql=_role_posture_sql(), + env=psql_env, + runner=runner, + ), + "runtime_role_posture", + ) + ) + stage_owner_role_posture = _assert_stage_owner_role_posture( + _parse_json_object( + _run_psql_success( + check="stage_owner_role_posture", + database=CANONICAL_DATABASE, + sql=_stage_owner_role_posture_sql(), + env=psql_env, + runner=runner, + ), + "stage_owner_role_posture", + ) + ) + function_privileges = _assert_function_privilege_posture( + _parse_json_object( + _run_psql_success( + check="function_privilege_posture", + database=CANONICAL_DATABASE, + sql=_function_privilege_posture_sql(), + env=psql_env, + runner=runner, + ), + "function_privilege_posture", + ) + ) + stage_function_definition = _assert_stage_function_definition( + _parse_json_object( + _run_psql_success( + check="stage_function_definition", + database=CANONICAL_DATABASE, + sql=_stage_function_definition_sql(), + env=psql_env, + runner=runner, + ), + "stage_function_definition", + ) + ) + catalog_privileges = _assert_catalog_privilege_posture( + _parse_json_object( + _run_psql_success( + check="catalog_privilege_posture", + database=CANONICAL_DATABASE, + sql=_catalog_privilege_posture_sql(), + env=psql_env, + runner=runner, + ), + "catalog_privilege_posture", + ) + ) + legacy_catalog_privileges = _assert_legacy_catalog_privilege_posture( + _parse_json_object( + _run_psql_success( + check="legacy_catalog_privileges", + database=LEGACY_DATABASE, + sql=_legacy_catalog_privilege_posture_sql(), + env=psql_env, + runner=runner, + ), + "legacy_catalog_privileges", + ) + ) + allowed_reads = _assert_allowed_reads( + _parse_json_object( + _run_psql_success( + check="canonical_allowed_reads", + database=CANONICAL_DATABASE, + sql=_allowed_read_sql(), + env=psql_env, + runner=runner, + ), + "canonical_allowed_reads", + ) + ) + rows_before = _parse_zero_count( + _run_psql_success( + check="canary_rows_before", + database=CANONICAL_DATABASE, + sql=_canary_count_sql(source_ref), + env=psql_env, + runner=runner, + ), + "canary_rows_before", + ) + staged = _assert_stage_result( + _parse_json_object( + _run_psql_success( + check="rolled_back_proposal_stage", + database=CANONICAL_DATABASE, + sql=_stage_sql(run_id, source_ref), + env=psql_env, + runner=runner, + ), + "rolled_back_proposal_stage", + ), + source_ref, + ) + rows_after = _parse_zero_count( + _run_psql_success( + check="canary_rows_after", + database=CANONICAL_DATABASE, + sql=_canary_count_sql(source_ref), + env=psql_env, + runner=runner, + ), + "canary_rows_after", + ) + + negative_permissions = [ + _expect_sqlstate(check, env=psql_env, runner=runner) for check in _negative_checks(run_id, source_ref) + ] + administrator_secret = _assert_administrator_secret_denied(env=gcloud_env, runner=runner) + + return { + "artifact": ARTIFACT_NAME, + "checks": { + "allowed_reads": allowed_reads, + "catalog_privileges": catalog_privileges, + "canary_rows_after": rows_after, + "canary_rows_before": rows_before, + "function_privileges": function_privileges, + "legacy_catalog_privileges": legacy_catalog_privileges, + "negative_permissions": negative_permissions, + "role_posture": role_posture, + "stage_function_definition": stage_function_definition, + "stage_owner_role_posture": stage_owner_role_posture, + "rolled_back_proposal": { + **staged, + "transaction": "rolled_back", + }, + }, + "current_tier": REQUIRED_TIER, + "database_identity": identity, + "execution": { + "canonical_writes_committed": False, + "service_independent": True, + "unix_user": RUNTIME_UNIX_USER, + }, + "generated_at_utc": generated_at, + "mode": "live_private_gcp_staging", + "required_tier": REQUIRED_TIER, + "run_id": run_id, + "schema_version": 1, + "secret_access": { + "administrator": { + **administrator_secret, + "secret": ADMINISTRATOR_PASSWORD_SECRET, + }, + "scoped_runtime": { + "result": "readable", + "secret": SCOPED_PASSWORD_SECRET, + "value_retained": False, + }, + }, + "status": "pass", + "safety": { + "canonical_write_committed": False, + "production_promotion_attempted": False, + "proposal_write_committed": False, + "secret_value_retained": False, + "telegram_send_attempted": False, + }, + "target": { + "database": CANONICAL_DATABASE, + "host": PRIVATE_CLOUDSQL_HOST, + "port": PRIVATE_CLOUDSQL_PORT, + "project": PROJECT_ID, + "role": RUNTIME_DATABASE_ROLE, + "sslmode": "require", + }, + } + + +def failure_receipt(run_id: str, error: VerificationError) -> dict[str, Any]: + return { + "artifact": ARTIFACT_NAME, + "current_tier": LOWER_TIER, + "error": error.as_dict(), + "generated_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"), + "required_tier": REQUIRED_TIER, + "run_id": run_id, + "schema_version": 1, + "status": "fail", + } + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n" + + +def write_receipt(path: Path, payload: Mapping[str, Any]) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + os.fchmod(descriptor, stat.S_IRUSR | stat.S_IWUSR) + with os.fdopen(descriptor, "w", encoding="utf-8", closefd=False) as handle: + handle.write(canonical_json(payload)) + handle.flush() + os.fsync(handle.fileno()) + finally: + os.close(descriptor) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-id", required=True, type=_run_id_arg) + parser.add_argument("--output", type=Path, help="optional sanitized JSON receipt (written mode 0600)") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + receipt = verify_runtime_permissions(args.run_id) + returncode = 0 + except VerificationError as exc: + receipt = failure_receipt(args.run_id, exc) + returncode = 1 + except Exception: + receipt = failure_receipt(args.run_id, VerificationError("internal_error", "verifier")) + returncode = 1 + + if args.output is not None: + try: + write_receipt(args.output, receipt) + except OSError: + receipt = failure_receipt(args.run_id, VerificationError("output_write_failed", "output")) + returncode = 1 + sys.stdout.write(canonical_json(receipt)) + return returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/verify_gcp_leoclean_service_environment.py b/ops/verify_gcp_leoclean_service_environment.py new file mode 100755 index 0000000..8dff5a1 --- /dev/null +++ b/ops/verify_gcp_leoclean_service_environment.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Verify the effective Leo service environment without exposing its values.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +PROJECT_ID = "teleo-501523" +PRIVATE_CLOUDSQL_HOST = "10.61.0.3" +PRIVATE_CLOUDSQL_PORT = "5432" +CANONICAL_DATABASE = "teleo_canonical" +RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime" +SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" +ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password" +RUNTIME_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +FORBIDDEN_EXACT_FIELDS = frozenset( + { + "ALL_PROXY", + "BASH_ENV", + "BASHOPTS", + "CDPATH", + "CURL_CA_BUNDLE", + "ENV", + "GLOBIGNORE", + "GOOGLE_APPLICATION_CREDENTIALS", + "HTTPS_PROXY", + "HTTP_PROXY", + "IFS", + "NO_PROXY", + "PS4", + "REQUESTS_CA_BUNDLE", + "SHELLOPTS", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SSLKEYLOGFILE", + } +) + + +class VerificationError(RuntimeError): + """A sanitized failure safe to serialize in a public receipt.""" + + def __init__(self, code: str, fields: tuple[str, ...] = ()) -> None: + self.code = code + self.fields = tuple(sorted(set(fields))) + super().__init__(code) + + def as_dict(self) -> dict[str, object]: + payload: dict[str, object] = {"code": self.code} + if self.fields: + payload["fields"] = list(self.fields) + return payload + + +class _SanitizedArgumentParser(argparse.ArgumentParser): + def error(self, _message: str) -> None: + raise VerificationError("invalid_arguments") + + +def validate_pid(value: str | int) -> int: + """Return a positive process id without retaining the input on failure.""" + + if isinstance(value, bool) or not isinstance(value, (str, int)): + raise VerificationError("invalid_pid") + if isinstance(value, str) and (not value.isascii() or not value.isdecimal() or value.startswith("0")): + raise VerificationError("invalid_pid") + try: + pid = int(value) + except (TypeError, ValueError): + raise VerificationError("invalid_pid") from None + if pid <= 0: + raise VerificationError("invalid_pid") + return pid + + +def expected_environment() -> dict[str, str]: + """Build the exact reviewed environment contract.""" + + return { + "PATH": RUNTIME_PATH, + "TELEO_CANONICAL_CLOUDSQL_DB": CANONICAL_DATABASE, + "TELEO_CLOUDSQL_DB": CANONICAL_DATABASE, + "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK": "0", + "TELEO_CLOUDSQL_HOST": PRIVATE_CLOUDSQL_HOST, + "TELEO_CLOUDSQL_PASSWORD_SECRET": SCOPED_PASSWORD_SECRET, + "TELEO_CLOUDSQL_PORT": PRIVATE_CLOUDSQL_PORT, + "TELEO_CLOUDSQL_USER": RUNTIME_DATABASE_ROLE, + "TELEO_GCP_METADATA_ONLY": "1", + "TELEO_GCP_PROJECT": PROJECT_ID, + "TELEO_KB_MODE": "cloudsql", + } + + +def parse_environment(raw: bytes) -> dict[str, str]: + """Parse a Linux ``/proc/PID/environ`` block and reject ambiguity.""" + + if not isinstance(raw, bytes) or not raw or not raw.endswith(b"\0"): + raise VerificationError("environment_malformed") + + records = raw[:-1].split(b"\0") + if any(not record for record in records): + raise VerificationError("environment_malformed") + + environment: dict[str, str] = {} + try: + for record in records: + encoded_key, separator, encoded_value = record.partition(b"=") + if not separator or not encoded_key: + raise VerificationError("environment_malformed") + key = encoded_key.decode("utf-8", "strict") + value = encoded_value.decode("utf-8", "strict") + if key in environment: + raise VerificationError("environment_malformed") + environment[key] = value + except UnicodeDecodeError: + raise VerificationError("environment_malformed") from None + return environment + + +def _forbidden_field_labels(environment: Mapping[str, str]) -> tuple[str, ...]: + labels: set[str] = set() + for key in environment: + normalized = key.upper() + if normalized.startswith("PG"): + labels.add("PG*") + if normalized.startswith("CLOUDSDK_"): + labels.add("CLOUDSDK_*") + if normalized.startswith("LD_"): + labels.add("LD_*") + if normalized.startswith("BASH_FUNC_"): + labels.add("BASH_FUNC_*") + if normalized.startswith("PYTHON") and normalized != "PYTHONUNBUFFERED": + labels.add("PYTHON*") + if normalized in FORBIDDEN_EXACT_FIELDS: + labels.add(normalized) + return tuple(sorted(labels)) + + +def validate_environment(environment: Mapping[str, str]) -> tuple[str, ...]: + """Validate the effective environment and return only reviewed field names.""" + + if not isinstance(environment, Mapping) or any( + not isinstance(key, str) or not isinstance(value, str) for key, value in environment.items() + ): + raise VerificationError("environment_malformed") + + expected = expected_environment() + administrator_secret = ADMINISTRATOR_PASSWORD_SECRET.casefold() + if any(administrator_secret in value.casefold() for value in environment.values()): + raise VerificationError("administrator_secret_reference") + + forbidden_fields = _forbidden_field_labels(environment) + if forbidden_fields: + raise VerificationError("forbidden_environment_fields", forbidden_fields) + + mismatched = tuple( + sorted(key for key, expected_value in expected.items() if environment.get(key) != expected_value) + ) + if mismatched: + raise VerificationError("environment_mismatch", mismatched) + + return tuple(sorted(expected)) + + +def verify_process_environment( + pid: int, + *, + proc_root: Path = Path("/proc"), +) -> dict[str, object]: + """Read and verify one process environment, returning a sanitized receipt.""" + + validated_pid = validate_pid(pid) + raw = (proc_root / str(validated_pid) / "environ").read_bytes() + validated_fields = validate_environment(parse_environment(raw)) + return { + "runtime_environment": "scoped", + "status": "pass", + "validated_fields": list(validated_fields), + } + + +def failure_receipt(error: VerificationError) -> dict[str, object]: + return {"error": error.as_dict(), "status": "fail"} + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n" + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = _SanitizedArgumentParser(description=__doc__, add_help=False) + parser.add_argument("--pid", required=True) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + receipt = verify_process_environment(validate_pid(args.pid)) + returncode = 0 + except VerificationError as exc: + receipt = failure_receipt(exc) + returncode = 1 + except Exception: + receipt = failure_receipt(VerificationError("internal_error")) + returncode = 1 + + sys.stdout.write(canonical_json(receipt)) + return returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_gcp_generated_db_direct_claim_suite.py b/scripts/run_gcp_generated_db_direct_claim_suite.py index f2a3e31..f335731 100644 --- a/scripts/run_gcp_generated_db_direct_claim_suite.py +++ b/scripts/run_gcp_generated_db_direct_claim_suite.py @@ -47,7 +47,7 @@ DEFAULT_HOST = "10.61.0.3" DEFAULT_PROJECT = "teleo-501523" DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql" -REVIEWED_CLOUDSQL_TOOL_SHA256 = "2617bb501456758131ff41a6c77b3c462e21c28c4348da319b60defb051c7c87" +REVIEWED_CLOUDSQL_TOOL_SHA256 = "66fa84b3bd6fe67e102fbce4509ea28c3537ba5bb1091e1a1a457b7425366064" REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a" COUNT_READBACK_RE = re.compile( r"DB readback:\s*claims:\s*`?(?P\d+)`?;\s*" diff --git a/systemd/leoclean-gcp-prod-parallel-cloudsql.conf b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf index 1b267f5..68d4d34 100644 --- a/systemd/leoclean-gcp-prod-parallel-cloudsql.conf +++ b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf @@ -1,9 +1,19 @@ [Service] UnsetEnvironment=PGPASSWORD -RuntimeDirectory=leoclean-gcp-prod-parallel -RuntimeDirectoryMode=0700 -Environment=CLOUDSDK_CONFIG=/run/leoclean-gcp-prod-parallel +UnsetEnvironment=PGPASSFILE PGSERVICE PGHOST PGPORT PGDATABASE PGUSER PGOPTIONS +UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN +UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS CDPATH GLOBIGNORE IFS PS4 +UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT LD_DEBUG LD_DEBUG_OUTPUT LD_DYNAMIC_WEAK LD_HWCAP_MASK LD_ORIGIN_PATH LD_PROFILE LD_SHOW_AUXV LD_TRACE_LOADED_OBJECTS LD_USE_LOAD_BIAS LD_VERBOSE LD_WARN +UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT PYTHONUSERBASE PYTHONHTTPSVERIFY +UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy +UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE +ReadOnlyPaths=/home/teleo/.hermes +ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state /home/teleo/.hermes/profiles/leoclean/workspace +BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin +CapabilityBoundingSet=~CAP_SYS_ADMIN +Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin Environment=TELEO_KB_MODE=cloudsql +Environment=TELEO_GCP_METADATA_ONLY=1 Environment=TELEO_GCP_PROJECT=teleo-501523 Environment=TELEO_CLOUDSQL_HOST=10.61.0.3 Environment=TELEO_CLOUDSQL_PORT=5432 diff --git a/tests/test_gcp_leoclean_runtime_permissions.py b/tests/test_gcp_leoclean_runtime_permissions.py new file mode 100644 index 0000000..ca97995 --- /dev/null +++ b/tests/test_gcp_leoclean_runtime_permissions.py @@ -0,0 +1,797 @@ +import hashlib +import json +import os +import stat +import subprocess +from collections.abc import Mapping +from pathlib import Path + +import pytest + +from ops import verify_gcp_leoclean_runtime_permissions as verifier + +RUN_ID = "20260715-leo-security" +SOURCE_REF = f"leo-runtime-permission:{RUN_ID}" +SECRET_VALUE = "unit-only-secret-value%42" +AGENT_ID = "11111111-1111-4111-8111-111111111111" + + +class FakeRunner: + def __init__( + self, + *, + sqlstate_override: tuple[str, str] | None = None, + sqlstate_missing_for: str | None = None, + unexpected_success_for: str | None = None, + scoped_failure: bool = False, + administrator_mode: str = "denied", + identity_override: Mapping[str, object] | None = None, + role_posture_override: Mapping[str, object] | None = None, + stage_owner_role_posture_override: Mapping[str, object] | None = None, + function_posture_override: Mapping[str, Mapping[str, object]] | None = None, + stage_definition_override: Mapping[str, object] | None = None, + catalog_posture_override: Mapping[str, object] | None = None, + legacy_catalog_posture_override: Mapping[str, object] | None = None, + ) -> None: + self.sqlstate_override = sqlstate_override + self.sqlstate_missing_for = sqlstate_missing_for + self.unexpected_success_for = unexpected_success_for + self.scoped_failure = scoped_failure + self.administrator_mode = administrator_mode + self.identity_override = dict(identity_override or {}) + self.role_posture_override = dict(role_posture_override or {}) + self.stage_owner_role_posture_override = dict(stage_owner_role_posture_override or {}) + self.function_posture_override = { + name: dict(values) for name, values in (function_posture_override or {}).items() + } + self.stage_definition_override = dict(stage_definition_override or {}) + self.catalog_posture_override = dict(catalog_posture_override or {}) + self.legacy_catalog_posture_override = dict(legacy_catalog_posture_override or {}) + self.calls: list[tuple[list[str], dict[str, str], int, bool]] = [] + self.negative = {check.sql: check for check in verifier._negative_checks(RUN_ID, SOURCE_REF)} + + def __call__( + self, + command: list[str], + env: Mapping[str, str], + timeout: int, + discard_stdout: bool, + ) -> verifier.CommandResult: + self.calls.append((list(command), dict(env), timeout, discard_stdout)) + + if command[0] == verifier.GCLOUD_BIN: + scoped = f"--secret={verifier.SCOPED_PASSWORD_SECRET}" in command + if scoped: + if self.scoped_failure: + return verifier.CommandResult(1, SECRET_VALUE, f"scoped failure {SECRET_VALUE}") + return verifier.CommandResult(0, f"{SECRET_VALUE}\n", f"ignored warning {SECRET_VALUE}") + assert f"--secret={verifier.ADMINISTRATOR_PASSWORD_SECRET}" in command + assert discard_stdout is True + if self.administrator_mode == "success": + return verifier.CommandResult(0, SECRET_VALUE, f"ignored {SECRET_VALUE}") + if self.administrator_mode == "wrong-error": + return verifier.CommandResult(1, SECRET_VALUE, f"network unavailable {SECRET_VALUE}") + return verifier.CommandResult( + 1, + SECRET_VALUE, + ( + "ERROR: PERMISSION_DENIED: Permission " + f"'{verifier.IAM_PERMISSION}' denied for the administrator secret; {SECRET_VALUE}" + ), + ) + + assert command[0] == verifier.PSQL_BIN + assert discard_stdout is False + sql = next(part.removeprefix("--command=") for part in command if part.startswith("--command=")) + + if sql == verifier._identity_sql(): + identity: dict[str, object] = { + "current_user": verifier.RUNTIME_DATABASE_ROLE, + "database": verifier.CANONICAL_DATABASE, + "leak": SECRET_VALUE, + "server_addr": verifier.PRIVATE_CLOUDSQL_HOST, + "server_port": verifier.PRIVATE_CLOUDSQL_PORT, + "session_user": verifier.RUNTIME_DATABASE_ROLE, + "ssl": True, + "ssl_version": "TLSv1.3", + } + identity.update(self.identity_override) + return verifier.CommandResult(0, json.dumps(identity), f"ignored {SECRET_VALUE}") + if sql == verifier._allowed_read_sql(): + return verifier.CommandResult( + 0, + json.dumps( + { + "claims_rows_sampled": 1, + "leak": SECRET_VALUE, + "proposal_rows_sampled": 0, + } + ), + f"ignored {SECRET_VALUE}", + ) + if sql == verifier._role_posture_sql(): + role_posture: dict[str, object] = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": True, + "can_replicate": False, + "connection_limit": 8, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "leak": SECRET_VALUE, + "role": verifier.RUNTIME_DATABASE_ROLE, + } + role_posture.update(self.role_posture_override) + return verifier.CommandResult(0, json.dumps(role_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._stage_owner_role_posture_sql(): + owner_posture: dict[str, object] = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": False, + "can_replicate": False, + "connection_limit": -1, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "leak": SECRET_VALUE, + "role": verifier.STAGE_OWNER_DATABASE_ROLE, + } + owner_posture.update(self.stage_owner_role_posture_override) + return verifier.CommandResult(0, json.dumps(owner_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._function_privilege_posture_sql(): + function_posture: dict[str, dict[str, object]] = { + name: { + "execute": expected_execute, + "exists": expected_exists, + "leak": SECRET_VALUE, + } + for name, _signature, expected_exists, expected_execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS + } + for name, override in self.function_posture_override.items(): + function_posture.setdefault(name, {}).update(override) + function_posture["unexpected_leak"] = {"value": SECRET_VALUE} + return verifier.CommandResult(0, json.dumps(function_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._stage_function_definition_sql(): + stage_definition: dict[str, object] = { + "acl_exact": True, + "argument_modes": None, + "configuration": ["search_path=pg_catalog, pg_temp"], + "exists": True, + "leak": SECRET_VALUE, + "runtime_execute": True, + "source": verifier.EXPECTED_STAGE_FUNCTION_SOURCE, + **dict(verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS), + } + stage_definition.update(self.stage_definition_override) + return verifier.CommandResult(0, json.dumps(stage_definition), f"ignored {SECRET_VALUE}") + if sql == verifier._catalog_privilege_posture_sql(): + catalog_posture: dict[str, object] = { + field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS + } + catalog_posture.update( + { + "leak": SECRET_VALUE, + **{field: True for field in verifier.CATALOG_TRUE_FIELDS}, + } + ) + catalog_posture.update(self.catalog_posture_override) + return verifier.CommandResult(0, json.dumps(catalog_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._legacy_catalog_privilege_posture_sql(): + legacy_posture: dict[str, object] = { + field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS + } + legacy_posture["leak"] = SECRET_VALUE + legacy_posture.update(self.legacy_catalog_posture_override) + return verifier.CommandResult(0, json.dumps(legacy_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._canary_count_sql(SOURCE_REF): + return verifier.CommandResult(0, "0\n", f"ignored {SECRET_VALUE}") + if sql == verifier._stage_sql(RUN_ID, SOURCE_REF): + return verifier.CommandResult( + 0, + json.dumps( + { + "agent_id_matches": True, + "proposal": { + "leak": SECRET_VALUE, + "proposed_by_agent_id": AGENT_ID, + "proposed_by_handle": "leo", + "source_ref": SOURCE_REF, + "status": "pending_review", + }, + } + ), + f"ignored {SECRET_VALUE}", + ) + + check = self.negative[sql] + if check.name == self.unexpected_success_for: + return verifier.CommandResult(0, SECRET_VALUE, f"unexpected success {SECRET_VALUE}") + if check.name == self.sqlstate_missing_for: + return verifier.CommandResult(1, SECRET_VALUE, f"denied without state {SECRET_VALUE}") + observed = check.expected_sqlstate + if self.sqlstate_override is not None and check.name == self.sqlstate_override[0]: + observed = self.sqlstate_override[1] + return verifier.CommandResult( + 1, + SECRET_VALUE, + f"ERROR: {observed}: expected test denial; {SECRET_VALUE}\nLOCATION: test", + ) + + +def poisoned_environment() -> dict[str, str]: + return { + "CLOUDSDK_AUTH_ACCESS_TOKEN": "must-not-reach-gcloud", + "GOOGLE_APPLICATION_CREDENTIALS": "/tmp/must-not-reach-gcloud.json", + "HOME": "/home/teleo", + "LANG": "C.UTF-8", + "LD_PRELOAD": "/tmp/must-not-load.so", + "OTHER": "preserved", + "PGDATABASE": "wrong-db", + "PGHOST": "public.example.invalid", + "PGPASSWORD": "administrator-password", + "PGSERVICE": "broad-service", + "pgsslmode": "disable", + } + + +def run_success(runner: FakeRunner) -> dict[str, object]: + return verifier.verify_runtime_permissions( + RUN_ID, + runner=runner, + base_env=poisoned_environment(), + effective_user="teleo", + ) + + +def assert_child_boundaries(runner: FakeRunner) -> None: + assert runner.calls + for command, env, timeout, discard_stdout in runner.calls: + assert SECRET_VALUE not in "\n".join(command) + assert timeout == verifier.COMMAND_TIMEOUT_SECONDS + pg_keys = sorted(key for key in env if key.upper().startswith("PG")) + if command[0] == verifier.PSQL_BIN: + assert pg_keys == ["PGPASSWORD"] + assert env["PGPASSWORD"] == SECRET_VALUE + assert discard_stdout is False + else: + assert pg_keys == [] + assert "CLOUDSDK_AUTH_ACCESS_TOKEN" not in env + assert "GOOGLE_APPLICATION_CREDENTIALS" not in env + assert env["HOME"] == "/home/teleo" + assert env["LANG"] == "C" + assert env["LC_ALL"] == "C" + assert env["PATH"] == verifier.CHILD_PATH + assert "LD_PRELOAD" not in env + assert "OTHER" not in env + + +def test_live_receipt_contract_is_sanitized_and_uses_exact_child_environments() -> None: + runner = FakeRunner() + + receipt = run_success(runner) + serialized = verifier.canonical_json(receipt) + + assert receipt["status"] == "pass" + assert receipt["current_tier"] == verifier.REQUIRED_TIER + assert receipt["database_identity"] == { + "current_user": verifier.RUNTIME_DATABASE_ROLE, + "database": verifier.CANONICAL_DATABASE, + "server_addr": verifier.PRIVATE_CLOUDSQL_HOST, + "server_port": verifier.PRIVATE_CLOUDSQL_PORT, + "session_user": verifier.RUNTIME_DATABASE_ROLE, + "ssl": True, + "ssl_version": "TLSv1.3", + } + assert receipt["checks"]["rolled_back_proposal"] == { + "agent_id_matches": True, + "proposed_by_agent_id": AGENT_ID, + "proposed_by_handle": "leo", + "source_ref": SOURCE_REF, + "status": "pending_review", + "transaction": "rolled_back", + } + assert receipt["checks"]["canary_rows_before"] == 0 + assert receipt["checks"]["canary_rows_after"] == 0 + assert receipt["checks"]["role_posture"] == { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": True, + "can_replicate": False, + "connection_limit": 8, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "role": verifier.RUNTIME_DATABASE_ROLE, + } + assert receipt["checks"]["stage_owner_role_posture"] == { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": False, + "can_replicate": False, + "connection_limit": -1, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "role": verifier.STAGE_OWNER_DATABASE_ROLE, + } + assert receipt["checks"]["function_privileges"] == { + name: { + "execute": expected_execute, + "exists": expected_exists, + "signature": signature, + } + for name, signature, expected_exists, expected_execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS + } + assert receipt["checks"]["catalog_privileges"] == { + **{field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS}, + **{field: True for field in verifier.CATALOG_TRUE_FIELDS}, + } + assert receipt["checks"]["legacy_catalog_privileges"] == { + field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS + } + assert receipt["checks"]["stage_function_definition"] == { + "acl_exact": True, + "argument_modes": None, + "configuration": ["search_path=pg_catalog, pg_temp"], + "exists": True, + **dict(verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS), + "runtime_execute": True, + "signature": verifier.STAGE_FUNCTION_SIGNATURE, + "source_sha256": verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256, + } + assert len(receipt["checks"]["negative_permissions"]) == len(verifier._negative_checks(RUN_ID, SOURCE_REF)) + assert receipt["secret_access"]["administrator"]["classification"] == "iam_permission_denied" + assert receipt["secret_access"]["administrator"]["stdout_discarded"] is True + assert SECRET_VALUE not in serialized + assert "administrator-password" not in serialized + assert "stage_leoclean_proposal is restricted" not in serialized + assert_child_boundaries(runner) + + administrator_call = runner.calls[-1] + assert administrator_call[0][0] == verifier.GCLOUD_BIN + assert administrator_call[3] is True + + +@pytest.mark.parametrize( + ("attribute", "unsafe_value"), + [ + ("can_login", False), + ("is_superuser", True), + ("can_create_db", True), + ("can_create_role", True), + ("inherits_privileges", True), + ("can_replicate", True), + ("bypasses_rls", True), + ("connection_limit", 7), + ], +) +def test_any_unsafe_runtime_role_attribute_fails_closed(attribute: str, unsafe_value: object) -> None: + runner = FakeRunner(role_posture_override={attribute: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "runtime_role_posture_mismatch" + assert caught.value.check == "runtime_role_posture" + assert SECRET_VALUE not in str(caught.value) + + +def test_any_direct_runtime_role_membership_edge_fails_closed() -> None: + runner = FakeRunner(role_posture_override={"direct_membership_edges": 1}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "runtime_role_posture_mismatch" + assert caught.value.check == "runtime_role_posture" + + +@pytest.mark.parametrize( + ("attribute", "unsafe_value"), + [ + ("role", "unexpected-owner"), + ("can_login", True), + ("is_superuser", True), + ("can_create_db", True), + ("can_create_role", True), + ("inherits_privileges", True), + ("can_replicate", True), + ("bypasses_rls", True), + ("connection_limit", 0), + ("direct_membership_edges", 1), + ], +) +def test_any_unsafe_stage_owner_role_posture_fails_closed(attribute: str, unsafe_value: object) -> None: + runner = FakeRunner(stage_owner_role_posture_override={attribute: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_owner_role_posture_mismatch" + assert caught.value.check == "stage_owner_role_posture" + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize( + "function_name", + ["approve_strict_proposal", "assert_approved_proposal", "finish_approved_proposal"], +) +def test_reviewer_or_apply_execute_grant_fails_closed(function_name: str) -> None: + runner = FakeRunner(function_posture_override={function_name: {"execute": True}}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "function_privilege_posture_mismatch" + assert caught.value.check == "function_privilege_posture" + + +@pytest.mark.parametrize( + "function_name", + [ + "stage_leoclean_proposal_5_arg", + "approve_strict_proposal", + "assert_approved_proposal", + "finish_approved_proposal", + ], +) +def test_missing_expected_exact_function_fails_closed(function_name: str) -> None: + runner = FakeRunner(function_posture_override={function_name: {"execute": False, "exists": False}}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "function_privilege_posture_mismatch" + assert caught.value.check == "function_privilege_posture" + + +def test_forged_overload_presence_or_missing_stage_execute_fails_closed() -> None: + forged = FakeRunner(function_posture_override={"stage_leoclean_proposal_6_arg": {"exists": True}}) + missing_execute = FakeRunner(function_posture_override={"stage_leoclean_proposal_5_arg": {"execute": False}}) + + for runner in (forged, missing_execute): + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + assert caught.value.code == "function_privilege_posture_mismatch" + + +def _drifted_value(expected: object) -> object: + if isinstance(expected, bool): + return not expected + if isinstance(expected, int): + return expected + 1 + if isinstance(expected, str): + return f"{expected}-drift" + if isinstance(expected, list): + return [*expected, "drift"] + raise AssertionError(f"unsupported test expectation type: {type(expected).__name__}") + + +@pytest.mark.parametrize( + ("field", "unsafe_value"), + [ + (field, _drifted_value(expected)) + for field, expected in verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS + ], +) +def test_any_stage_function_metadata_drift_fails_closed(field: str, unsafe_value: object) -> None: + runner = FakeRunner(stage_definition_override={field: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_function_definition_mismatch" + assert caught.value.check == "stage_function_definition" + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize( + ("field", "unsafe_value"), + [ + ("exists", False), + ("argument_modes", ["i", "i", "i", "i", "i"]), + ("configuration", ["search_path=public"]), + ("acl_exact", False), + ("runtime_execute", False), + ("leakproof", 0), + ("argument_defaults", False), + ], +) +def test_any_stage_function_contract_drift_fails_closed(field: str, unsafe_value: object) -> None: + runner = FakeRunner(stage_definition_override={field: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_function_definition_mismatch" + assert caught.value.check == "stage_function_definition" + + +@pytest.mark.parametrize( + "unsafe_source", + [ + None, + 42, + verifier.EXPECTED_STAGE_FUNCTION_SOURCE.replace("return to_jsonb(staged);", "return '{}'::jsonb;"), + "x" * (verifier.MAX_STAGE_FUNCTION_SOURCE_BYTES + 1), + ], +) +def test_missing_malformed_oversized_or_semantically_drifted_stage_body_fails_closed( + unsafe_source: object, +) -> None: + runner = FakeRunner(stage_definition_override={"source": unsafe_source}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_function_definition_mismatch" + assert caught.value.check == "stage_function_definition" + failure = verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value)) + assert "stage_leoclean_proposal is restricted" not in failure + assert SECRET_VALUE not in failure + + +def test_stage_body_normalization_accepts_only_line_ending_and_framing_newline_changes() -> None: + crlf_source = "\r\n" + verifier.EXPECTED_STAGE_FUNCTION_SOURCE.strip("\n").replace("\n", "\r\n") + "\r\n" + + receipt = run_success(FakeRunner(stage_definition_override={"source": crlf_source})) + + assert ( + receipt["checks"]["stage_function_definition"]["source_sha256"] + == verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256 + ) + + +def test_embedded_stage_body_attestation_is_tied_to_the_provisioning_sql_source() -> None: + provisioning_sql = ( + Path(__file__).resolve().parents[1] / "ops" / "gcp_leoclean_runtime_role.sql" + ).read_text(encoding="utf-8") + function_anchor = "create or replace function kb_stage.stage_leoclean_proposal(" + assert provisioning_sql.count(function_anchor) == 1 + function_section = provisioning_sql.split(function_anchor, 1)[1] + assert function_section.count("as $function$") == 1 + provisioned_source = function_section.split("as $function$", 1)[1].split("$function$;", 1)[0] + + assert provisioned_source == verifier.EXPECTED_STAGE_FUNCTION_SOURCE + assert ( + hashlib.sha256(verifier.normalize_stage_function_source(provisioned_source).encode("utf-8")).hexdigest() + == verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256 + ) + + +def test_catalog_queries_use_exact_regprocedure_signatures_and_both_membership_directions() -> None: + function_sql = verifier._function_privilege_posture_sql() + role_sql = verifier._role_posture_sql() + owner_role_sql = verifier._stage_owner_role_posture_sql() + stage_definition_sql = verifier._stage_function_definition_sql() + catalog_sql = verifier._catalog_privilege_posture_sql() + legacy_sql = verifier._legacy_catalog_privilege_posture_sql() + + assert "pg_catalog.to_regprocedure(signature)" in function_sql + for _name, signature, _exists, _execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS: + assert signature in function_sql + assert "membership.roleid = role_row.oid" in role_sql + assert "membership.member = role_row.oid" in role_sql + assert "membership.roleid = role_row.oid" in owner_role_sql + assert "membership.member = role_row.oid" in owner_role_sql + assert verifier.STAGE_OWNER_DATABASE_ROLE in owner_role_sql + assert verifier.STAGE_FUNCTION_SIGNATURE in stage_definition_sql + assert "left join pg_catalog.pg_proc" in stage_definition_sql + assert "function_catalog.prosrc" in stage_definition_sql + assert "function_catalog.prokind" in stage_definition_sql + assert "function_catalog.proargtypes" in stage_definition_sql + assert "function_catalog.prosecdef" in stage_definition_sql + assert "pg_catalog.pg_get_function_result" in stage_definition_sql + assert "pg_catalog.aclexplode" in stage_definition_sql + assert "pg_catalog.pg_shdepend" in catalog_sql + assert "pg_catalog.has_table_privilege" in catalog_sql + assert "pg_catalog.has_column_privilege" in catalog_sql + assert "pg_catalog.has_sequence_privilege" in catalog_sql + assert "pg_catalog.has_function_privilege" in catalog_sql + assert "pg_catalog.aclexplode" in catalog_sql + assert "nspname !~ '^pg_'" in catalog_sql + assert "nspname <> 'information_schema'" in catalog_sql + assert verifier.STAGE_OWNER_DATABASE_ROLE in catalog_sql + assert "teleo_restore" in legacy_sql + assert "pg_catalog.has_schema_privilege" in legacy_sql + assert "pg_catalog.has_table_privilege" in legacy_sql + assert "pg_catalog.has_column_privilege" in legacy_sql + assert "pg_catalog.has_sequence_privilege" in legacy_sql + assert "pg_catalog.has_function_privilege" in legacy_sql + for schema, relation in verifier.READ_TABLE_ALLOWLIST: + assert schema in catalog_sql + assert relation in catalog_sql + + +@pytest.mark.parametrize("field", verifier.CATALOG_ZERO_COUNT_FIELDS) +def test_any_unexpected_catalog_privilege_fails_closed(field: str) -> None: + runner = FakeRunner(catalog_posture_override={field: 1}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "catalog_privilege_posture_mismatch" + assert caught.value.check == "catalog_privilege_posture" + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize("field", verifier.CATALOG_TRUE_FIELDS) +def test_any_missing_required_catalog_privilege_fails_closed(field: str) -> None: + runner = FakeRunner(catalog_posture_override={field: False}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "catalog_privilege_posture_mismatch" + + +@pytest.mark.parametrize("field", verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS) +def test_any_teleo_restore_catalog_privilege_fails_closed(field: str) -> None: + runner = FakeRunner(legacy_catalog_posture_override={field: 1}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "legacy_catalog_privilege_posture_mismatch" + assert caught.value.check == "legacy_catalog_privileges" + assert SECRET_VALUE not in str(caught.value) + + +def test_teleo_restore_catalog_and_behavioral_denial_are_both_exercised_on_legacy_database() -> None: + runner = FakeRunner() + + receipt = run_success(runner) + + legacy_catalog_call = next( + command + for command, _env, _timeout, _discard in runner.calls + if any(part == f"--command={verifier._legacy_catalog_privilege_posture_sql()}" for part in command) + ) + assert any(f"dbname={verifier.LEGACY_DATABASE}" in part for part in legacy_catalog_call) + legacy_denial = next( + check for check in receipt["checks"]["negative_permissions"] if check["check"] == "legacy_teleo_restore_read" + ) + assert legacy_denial == { + "check": "legacy_teleo_restore_read", + "expected_sqlstate": "42501", + "observed_sqlstate": "42501", + "result": "denied", + } + + +def test_required_sqlstate_mismatch_fails_closed_without_secret_in_error() -> None: + runner = FakeRunner(sqlstate_override=("canonical_insert", "23505")) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + error = caught.value + assert error.code == "negative_permission_sqlstate_mismatch" + assert error.check == "canonical_insert" + assert error.expected_sqlstate == "42501" + assert error.observed_sqlstate == "23505" + assert SECRET_VALUE not in str(error) + assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, error)) + assert_child_boundaries(runner) + + +def test_missing_verbose_sqlstate_fails_closed_without_raw_stderr() -> None: + runner = FakeRunner(sqlstate_missing_for="forged_proposer_overload") + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + error = caught.value + assert error.code == "negative_permission_sqlstate_missing" + assert error.expected_sqlstate == "42883" + assert error.observed_sqlstate == "missing" + assert SECRET_VALUE not in json.dumps(error.as_dict()) + + +def test_unexpected_permission_success_fails_closed_and_transaction_sql_has_rollback() -> None: + runner = FakeRunner(unexpected_success_for="proposal_update") + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "negative_permission_unexpectedly_succeeded" + assert caught.value.observed_sqlstate == "success" + proposal_update = next( + check for check in verifier._negative_checks(RUN_ID, SOURCE_REF) if check.name == "proposal_update" + ) + assert proposal_update.sql.startswith("begin;\n") + assert proposal_update.sql.endswith("\nrollback;") + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize("administrator_mode", ["success", "wrong-error"]) +def test_administrator_secret_must_be_exact_iam_denial_and_stdout_is_discarded(administrator_mode: str) -> None: + runner = FakeRunner(administrator_mode=administrator_mode) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + expected_code = ( + "administrator_secret_unexpectedly_readable" + if administrator_mode == "success" + else "administrator_secret_denial_not_iam_permission" + ) + assert caught.value.code == expected_code + assert SECRET_VALUE not in str(caught.value) + assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value)) + assert runner.calls[-1][3] is True + + +def test_scoped_secret_failure_never_repeats_command_output() -> None: + runner = FakeRunner(scoped_failure=True) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "scoped_secret_access_failed" + assert SECRET_VALUE not in str(caught.value) + assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value)) + assert len(runner.calls) == 1 + + +@pytest.mark.parametrize( + "invalid", + ["short", "UPPERCASE-RUN", "contains_underscore", "contains.dot", " leading-space", "a" * 65], +) +def test_run_id_validation_is_strict(invalid: str) -> None: + with pytest.raises(ValueError, match="8-64 lowercase"): + verifier.validate_run_id(invalid) + + +def test_wrong_identity_or_non_ssl_connection_fails_before_permission_probes() -> None: + runner = FakeRunner(identity_override={"ssl": False}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "private_ssl_identity_mismatch" + assert len(runner.calls) == 2 + assert SECRET_VALUE not in str(caught.value) + + +def test_optional_receipt_is_canonical_mode_0600_and_refuses_symlink(tmp_path: Path) -> None: + payload = {"z": 2, "a": {"safe": True}} + output = tmp_path / "receipt.json" + + verifier.write_receipt(output, payload) + + assert output.read_text(encoding="utf-8") == verifier.canonical_json(payload) + assert stat.S_IMODE(output.stat().st_mode) == 0o600 + assert output.read_text(encoding="utf-8").startswith('{"a":') + + target = tmp_path / "target.json" + target.write_text("untouched", encoding="utf-8") + symlink = tmp_path / "symlink.json" + symlink.symlink_to(target) + with pytest.raises(OSError): + verifier.write_receipt(symlink, payload) + assert target.read_text(encoding="utf-8") == "untouched" + + +def test_subprocess_runner_discards_administrator_stdout_at_file_descriptor(monkeypatch: pytest.MonkeyPatch) -> None: + observed: dict[str, object] = {} + + def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + observed["command"] = command + observed.update(kwargs) + return subprocess.CompletedProcess(command, 1, stdout=None, stderr="PERMISSION_DENIED") + + monkeypatch.setattr(verifier.subprocess, "run", fake_run) + + result = verifier.subprocess_runner([verifier.GCLOUD_BIN, "test"], os.environ, 3, True) + + assert observed["stdout"] is subprocess.DEVNULL + assert observed["stdin"] is subprocess.DEVNULL + assert observed["stderr"] is subprocess.PIPE + assert result.stdout == "" + assert result.returncode == 1 diff --git a/tests/test_gcp_leoclean_service_environment.py b/tests/test_gcp_leoclean_service_environment.py new file mode 100644 index 0000000..a1be9e6 --- /dev/null +++ b/tests/test_gcp_leoclean_service_environment.py @@ -0,0 +1,281 @@ +import json +import os +import subprocess +import sys +from collections.abc import Mapping +from pathlib import Path + +import pytest + +from ops import verify_gcp_leoclean_service_environment as verifier + + +def environment_block(environment: Mapping[str, str]) -> bytes: + return b"\0".join(f"{key}={value}".encode() for key, value in environment.items()) + b"\0" + + +def test_all_expected_fields_pass_and_receipt_contains_no_values() -> None: + environment = verifier.expected_environment() + environment.update({"HOME": "/home/teleo", "LANG": "C.UTF-8"}) + + parsed = verifier.parse_environment(environment_block(environment)) + validated = verifier.validate_environment(parsed) + receipt = { + "runtime_environment": "scoped", + "status": "pass", + "validated_fields": list(validated), + } + rendered = verifier.canonical_json(receipt) + + assert validated == tuple(sorted(verifier.expected_environment())) + assert json.loads(rendered) == receipt + assert rendered == json.dumps(receipt, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n" + for value in verifier.expected_environment().values(): + assert value not in rendered + + +@pytest.mark.parametrize("field", sorted(verifier.expected_environment())) +def test_each_expected_field_mismatch_fails_closed(field: str) -> None: + environment = verifier.expected_environment() + environment[field] = "wrong-value-that-is-not-a-secret" + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + assert caught.value.code == "environment_mismatch" + assert caught.value.fields == (field,) + + +@pytest.mark.parametrize( + ("field", "unsafe_value", "expected_code"), + [ + ("TELEO_CLOUDSQL_USER", "postgres", "environment_mismatch"), + ( + "TELEO_CLOUDSQL_PASSWORD_SECRET", + verifier.ADMINISTRATOR_PASSWORD_SECRET, + "administrator_secret_reference", + ), + ], +) +def test_later_dropin_effective_override_fails_without_outputting_value( + field: str, + unsafe_value: str, + expected_code: str, +) -> None: + environment = verifier.expected_environment() + environment[field] = unsafe_value + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert caught.value.code == expected_code + assert unsafe_value not in rendered + + +@pytest.mark.parametrize( + ("field", "safe_label"), + [ + ("PGPASSWORD", "PG*"), + ("pgservice", "PG*"), + ("CLOUDSDK_CONFIG", "CLOUDSDK_*"), + ("CLOUDSDK_CORE_PROJECT", "CLOUDSDK_*"), + ("CLOUDSDK_AUTH_ACCESS_TOKEN", "CLOUDSDK_*"), + ("cloudsdk_auth_refresh_token", "CLOUDSDK_*"), + ("GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_APPLICATION_CREDENTIALS"), + ("LD_PRELOAD", "LD_*"), + ("LD_LIBRARY_PATH", "LD_*"), + ("ld_audit", "LD_*"), + ("BASH_ENV", "BASH_ENV"), + ("ENV", "ENV"), + ("BASH_FUNC_injected%%", "BASH_FUNC_*"), + ("PYTHONHOME", "PYTHON*"), + ("PYTHONPATH", "PYTHON*"), + ("PYTHONHTTPSVERIFY", "PYTHON*"), + ("pythonstartup", "PYTHON*"), + ("HTTP_PROXY", "HTTP_PROXY"), + ("https_proxy", "HTTPS_PROXY"), + ("ALL_PROXY", "ALL_PROXY"), + ("no_proxy", "NO_PROXY"), + ("SSL_CERT_FILE", "SSL_CERT_FILE"), + ("ssl_cert_dir", "SSL_CERT_DIR"), + ("REQUESTS_CA_BUNDLE", "REQUESTS_CA_BUNDLE"), + ("curl_ca_bundle", "CURL_CA_BUNDLE"), + ("SSLKEYLOGFILE", "SSLKEYLOGFILE"), + ], +) +def test_environment_file_broader_credentials_and_forbidden_fields_fail_closed( + field: str, + safe_label: str, +) -> None: + environment = verifier.expected_environment() + injected_value = "sensitive-injected-value" + environment[field] = injected_value + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert caught.value.code == "forbidden_environment_fields" + assert caught.value.fields == (safe_label,) + assert injected_value not in rendered + + +def test_arbitrary_forbidden_key_suffix_is_not_an_output_channel() -> None: + environment = verifier.expected_environment() + environment["PG_SECRET_MARKER"] = "not-output" + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert "PG_SECRET_MARKER" not in rendered + assert "PG*" in rendered + + +def test_pythonunbuffered_is_the_only_allowed_python_runtime_tuning() -> None: + environment = verifier.expected_environment() + environment["PYTHONUNBUFFERED"] = "1" + + assert verifier.validate_environment(environment) == tuple(sorted(verifier.expected_environment())) + + +def test_bash_env_executes_before_a_script_but_effective_verifier_rejects_it(tmp_path: Path) -> None: + marker = "BASH_ENV_EXECUTED" + payload = tmp_path / "bash-env.sh" + payload.write_text(f"printf '{marker}\\n'\n", encoding="utf-8") + completed = subprocess.run( + ["/bin/bash", "-c", ":"], + env={"BASH_ENV": str(payload), "PATH": verifier.RUNTIME_PATH}, + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 0 + assert completed.stdout.strip() == marker + + environment = verifier.expected_environment() + environment["BASH_ENV"] = str(payload) + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + assert caught.value.fields == ("BASH_ENV",) + + +def test_admin_secret_reference_in_any_value_fails_without_field_or_value_leak() -> None: + environment = verifier.expected_environment() + environment["UNRELATED"] = f"prefix/{verifier.ADMINISTRATOR_PASSWORD_SECRET}/suffix" + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert caught.value.code == "administrator_secret_reference" + assert verifier.ADMINISTRATOR_PASSWORD_SECRET not in rendered + assert "UNRELATED" not in rendered + + +@pytest.mark.parametrize( + "raw", + [ + b"A=1", + b"A=1\0\0", + b"A=1\0BROKEN\0", + b"=value\0", + b"A=1\0A=2\0", + b"A=\xff\0", + b"", + ], +) +def test_malformed_nul_environment_is_rejected(raw: bytes) -> None: + with pytest.raises(verifier.VerificationError) as caught: + verifier.parse_environment(raw) + + assert caught.value.code == "environment_malformed" + + +def test_parser_preserves_equals_inside_values() -> None: + assert verifier.parse_environment(b"A=one=two\0B=\0") == {"A": "one=two", "B": ""} + + +@pytest.mark.parametrize("value", [0, -1, 1.5, "0", "01", "+1", "-3", "not-a-pid", True]) +def test_pid_must_be_a_positive_integer(value: object) -> None: + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_pid(value) # type: ignore[arg-type] + + assert caught.value.code == "invalid_pid" + + +def test_cli_errors_are_canonical_sanitized_json_and_exit_one(capsys: pytest.CaptureFixture[str]) -> None: + secret_input = "not-a-pid-secret-marker" + + returncode = verifier.main(["--pid", secret_input]) + + output = capsys.readouterr() + assert returncode == 1 + assert output.err == "" + assert json.loads(output.out) == {"error": {"code": "invalid_pid"}, "status": "fail"} + assert secret_input not in output.out + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="requires Linux /proc/PID/environ") +def test_cli_reads_real_process_environment_and_detects_effective_conflict( + capsys: pytest.CaptureFixture[str], +) -> None: + expected = verifier.expected_environment() + good_process = subprocess.Popen( + ["/bin/sleep", "30"], + env=expected, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + assert verifier.main(["--pid", str(good_process.pid)]) == 0 + good_output = capsys.readouterr() + assert good_output.err == "" + assert json.loads(good_output.out)["status"] == "pass" + finally: + good_process.terminate() + good_process.wait(timeout=5) + + conflicting = dict(expected) + conflicting["TELEO_CLOUDSQL_USER"] = "postgres" + conflicting["GOOGLE_APPLICATION_CREDENTIALS"] = "/tmp/broader-credential.json" + bad_process = subprocess.Popen( + ["/bin/sleep", "30"], + env=conflicting, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + assert verifier.main(["--pid", str(bad_process.pid)]) == 1 + bad_output = capsys.readouterr() + receipt = json.loads(bad_output.out) + assert bad_output.err == "" + assert receipt == { + "error": {"code": "forbidden_environment_fields", "fields": ["GOOGLE_APPLICATION_CREDENTIALS"]}, + "status": "fail", + } + assert "postgres" not in bad_output.out + assert "/tmp/broader-credential.json" not in bad_output.out + finally: + bad_process.terminate() + bad_process.wait(timeout=5) + + +def test_unexpected_read_error_is_generic_and_does_not_echo_path( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + sensitive_marker = "sensitive-proc-read-detail" + + def fail_read(_self: object) -> bytes: + raise OSError(sensitive_marker) + + monkeypatch.setattr(verifier.Path, "read_bytes", fail_read) + + assert verifier.main(["--pid", str(os.getpid())]) == 1 + output = capsys.readouterr() + assert json.loads(output.out) == {"error": {"code": "internal_error"}, "status": "fail"} + assert sensitive_marker not in output.out diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index 169f0c1..3a41541 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import base64 import contextlib import importlib.util import io @@ -64,6 +65,14 @@ def test_cloudsql_wrapper_supports_expected_review_gated_commands() -> None: assert command in cloudsql_text assert "status" in vps_text + assert "SCRIPT_DIR=${SCRIPT_PATH%/*}" in wrapper_text + assert 'SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"' in wrapper_text + assert 'CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py"' in wrapper_text + assert wrapper_text.startswith("#!/bin/bash\n") + assert "exec /usr/bin/python3" in wrapper_text + assert "command -v psql" in wrapper_text + assert "gcloud" not in wrapper_text + assert '"gcloud"' not in cloudsql_text assert "kb_stage.kb_proposals" in cloudsql_text assert "canonical apply" in cloudsql_text.lower() @@ -95,7 +104,7 @@ def _install_wrapper_fixture(tmp_path: Path, *, missing: frozenset[str] = frozen target = shutil.which(name) assert target (fake_bin / name).symlink_to(target) - for name in ("psql", "gcloud"): + for name in ("psql",): if name in missing: continue executable = fake_bin / name @@ -146,7 +155,7 @@ def test_cloudsql_wrapper_auto_mode_never_falls_back_after_supported_command_fai assert "local" not in completed.stdout -@pytest.mark.parametrize("missing", ["cloudsql-tool", "psql", "gcloud"]) +@pytest.mark.parametrize("missing", ["cloudsql-tool", "psql"]) def test_cloudsql_wrapper_auto_mode_fails_closed_when_prerequisite_is_missing(tmp_path: Path, missing: str) -> None: wrapper, env = _install_wrapper_fixture(tmp_path, missing=frozenset({missing})) @@ -209,6 +218,72 @@ def test_bridge_source_does_not_commit_raw_secret_values() -> None: assert not re.search(pattern, combined), pattern +def _runtime_connection_args(module, **overrides): + values = { + "credential_mode": "runtime", + "user": module.RUNTIME_DB_USER, + "password_secret": module.RUNTIME_PASSWORD_SECRET, + "project": module.RUNTIME_GCP_PROJECT, + "host": module.RUNTIME_CLOUDSQL_HOST, + "port": module.RUNTIME_CLOUDSQL_PORT, + "db": module.RUNTIME_CLOUDSQL_DB, + "canonical_db": module.RUNTIME_CLOUDSQL_DB, + "command": "status", + } + values.update(overrides) + return SimpleNamespace(**values) + + +def _configure_runtime_environment(monkeypatch: pytest.MonkeyPatch) -> None: + for key in tuple(os.environ): + normalized_key = key.upper() + if ( + normalized_key.startswith("PG") + or normalized_key.startswith("CLOUDSDK_") + or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS" + or normalized_key in {"LD_PRELOAD", "PYTHONHOME", "PYTHONPATH"} + or normalized_key == "PROXY" + or normalized_key.endswith("_PROXY") + or normalized_key in { + "CURL_CA_BUNDLE", + "PYTHONHTTPSVERIFY", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SSLKEYLOGFILE", + } + ): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("TELEO_GCP_METADATA_ONLY", "1") + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") + + +class _FakeHttpResponse: + def __init__(self, body: bytes, status: int = 200) -> None: + self.body = body + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + def getcode(self) -> int: + return self.status + + def read(self, _limit: int) -> bytes: + return self.body + + +def _runtime_http_bodies(module, password: bytes = b"scoped-runtime-password") -> list[bytes]: + return [ + module.RUNTIME_SERVICE_ACCOUNT.encode("ascii"), + json.dumps({"access_token": "metadata-access-token", "token_type": "Bearer", "expires_in": 300}).encode(), + json.dumps({"payload": {"data": base64.b64encode(password).decode("ascii")}}).encode(), + ] + + def test_cloudsql_runtime_requires_the_exact_scoped_identity(monkeypatch: pytest.MonkeyPatch) -> None: module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") monkeypatch.delenv("PGPASSWORD", raising=False) @@ -249,17 +324,113 @@ def test_cloudsql_runtime_accepts_only_scoped_credentials_and_rejects_inherited_ monkeypatch: pytest.MonkeyPatch, ) -> None: module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") - args = SimpleNamespace( - credential_mode="runtime", - user=module.RUNTIME_DB_USER, - password_secret=module.RUNTIME_PASSWORD_SECRET, - ) - monkeypatch.delenv("PGPASSWORD", raising=False) + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) module.validate_runtime_connection(args) monkeypatch.setenv("PGPASSWORD", "") - with pytest.raises(SystemExit, match="Refusing inherited PGPASSWORD"): + with pytest.raises(SystemExit, match="Refusing inherited credential, proxy, TLS, or PostgreSQL environment"): + module.validate_runtime_connection(args) + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("GOOGLE_APPLICATION_CREDENTIALS", "/tmp/ambient.json"), + ("google_application_credentials", "/tmp/ambient-lowercase.json"), + ("CLOUDSDK_CONFIG", "/tmp/gcloud"), + ("CLOUDSDK_AUTH_ACCESS_TOKEN", "ambient-token"), + ("cloudsdk_core_project", "attacker-project"), + ("cloudsdk_auth_access_token", "ambient-lowercase-token"), + ("PGHOST", "attacker.invalid"), + ("PGOPTIONS", "-c search_path=attacker"), + ("pgservice", "attacker-service"), + ("ld_preload", "/tmp/attacker.so"), + ("LD_LIBRARY_PATH", "/tmp/attacker-library"), + ("Ld_AuDiT", "/tmp/attacker-audit.so"), + ("BASH_ENV", "/tmp/attacker-bash-env"), + ("ENV", "/tmp/attacker-sh-env"), + ("BASH_FUNC_psql%%", "() { :; }"), + ("PythonHome", "/tmp/attacker-python"), + ("pythonpath", "/tmp/attacker-modules"), + ("PYTHONHTTPSVERIFY", "0"), + ("pythonstartup", "/tmp/attacker-startup.py"), + ("https_proxy", "http://attacker.invalid:8080"), + ("HtTp_PrOxY", "http://attacker.invalid:8080"), + ("ssl_cert_file", "/tmp/attacker.pem"), + ("SSL_CERT_DIR", "/tmp/attacker-certs"), + ("REQUESTS_CA_BUNDLE", "/tmp/attacker.pem"), + ("SSLKEYLOGFILE", "/tmp/attacker-tls-keys"), + ], +) +def test_cloudsql_runtime_rejects_ambient_auth_and_pg_environment( + monkeypatch: pytest.MonkeyPatch, + name: str, + value: str, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.setenv(name, value) + + with pytest.raises(SystemExit, match="Refusing inherited credential, proxy, TLS, or PostgreSQL environment"): + module.validate_runtime_connection(args) + + +def test_cloudsql_runtime_allows_only_pythonunbuffered_runtime_tuning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.setenv("PYTHONUNBUFFERED", "1") + + module.validate_runtime_connection(args) + + +def test_cloudsql_runtime_requires_explicit_metadata_only_mode(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.delenv("TELEO_GCP_METADATA_ONLY") + + with pytest.raises(SystemExit, match="TELEO_GCP_METADATA_ONLY=1"): + module.validate_runtime_connection(args) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("project", "other-project", "project=teleo-501523"), + ("host", "127.0.0.1", "host=10.61.0.3"), + ("port", "6543", "port=5432"), + ("db", "teleo_kb", "db=teleo_canonical"), + ("canonical_db", "teleo_kb", "canonical_db=teleo_canonical"), + ], +) +def test_cloudsql_runtime_rejects_any_target_drift_before_credentials( + monkeypatch: pytest.MonkeyPatch, + field: str, + value: str, + message: str, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module, **{field: value}) + monkeypatch.delenv("PGPASSWORD", raising=False) + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") + + with pytest.raises(SystemExit, match=message): + module.validate_runtime_connection(args) + + +def test_cloudsql_runtime_rejects_audit_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + monkeypatch.delenv("PGPASSWORD", raising=False) + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "1") + + with pytest.raises(SystemExit, match="AUDIT_FALLBACK=0"): module.validate_runtime_connection(args) @@ -290,33 +461,44 @@ def test_cloudsql_clone_mode_is_read_only_and_bound_to_one_disposable_database( module.validate_runtime_connection(args) -def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pytest.MonkeyPatch) -> None: +def test_cloudsql_clone_mode_never_resolves_a_password_secret_argument(monkeypatch: pytest.MonkeyPatch) -> None: module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") args = SimpleNamespace( - credential_mode="runtime", - user=module.RUNTIME_DB_USER, - password_secret=module.RUNTIME_PASSWORD_SECRET, - host="10.61.0.3", - port="5432", - db="teleo_canonical", - canonical_db="teleo_canonical", - project="teleo-501523", + credential_mode="clone-readonly", + user="postgres", + password_secret="gcp-teleo-pgvector-standby-postgres-password", + canonical_db="teleo_clone_canary_123", + db="teleo_clone_canary_123", command="status", ) monkeypatch.delenv("PGPASSWORD", raising=False) - monkeypatch.setenv("PGHOST", "attacker.invalid") - monkeypatch.setenv("PGSERVICE", "untrusted-service") - monkeypatch.setenv("PGPASSFILE", "/tmp/untrusted-pgpass") - monkeypatch.setenv("PGOPTIONS", "-c search_path=attacker") + monkeypatch.setenv("PGOPTIONS", module.CLONE_READ_ONLY_PGOPTIONS) + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_args, **_kwargs: pytest.fail("a password-secret argument must never launch a credential helper"), + ) + + with pytest.raises(SystemExit, match="requires an explicit user and inherited test credential"): + module.password(args) + + +def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.setenv("HOME", "/tmp/untrusted-home") + monkeypatch.setenv("LANG", "attacker-locale") + monkeypatch.setenv("SHELL", "/tmp/untrusted-shell") + monkeypatch.setenv("UNRELATED_AMBIENT_VALUE", "must-not-reach-psql") captured: dict[str, object] = {} def fake_run(command, **kwargs): - if command[0] == "gcloud": - return SimpleNamespace(returncode=0, stdout="scoped-runtime-password\n", stderr="") captured["command"] = command captured["kwargs"] = kwargs return SimpleNamespace(returncode=0, stdout="ok\n", stderr="") + monkeypatch.setattr(module, "runtime_password_from_metadata", lambda: "scoped-runtime-password") monkeypatch.setattr(module.subprocess, "run", fake_run) assert module.run_psql(args, "select 1;") == "ok\n" @@ -324,16 +506,279 @@ def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pyt kwargs = captured["kwargs"] assert isinstance(kwargs, dict) env = kwargs["env"] - assert env["PGHOST"] == "10.61.0.3" - assert env["PGPORT"] == "5432" - assert env["PGDATABASE"] == "teleo_canonical" - assert env["PGUSER"] == module.RUNTIME_DB_USER - assert env["PGSSLMODE"] == "require" - assert env["PGCONNECT_TIMEOUT"] == "8" - assert env["PGPASSWORD"] == "scoped-runtime-password" - assert "PGOPTIONS" not in env - assert "PGSERVICE" not in env - assert "PGPASSFILE" not in env + assert env == { + "PATH": "/usr/bin:/bin", + "PGHOST": "10.61.0.3", + "PGPORT": "5432", + "PGDATABASE": "teleo_canonical", + "PGUSER": module.RUNTIME_DB_USER, + "PGSSLMODE": "require", + "PGCONNECT_TIMEOUT": "8", + "PGPASSWORD": "scoped-runtime-password", + } + + +def test_cloudsql_runtime_metadata_and_secret_requests_are_exact_and_silent( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + bodies = _runtime_http_bodies(module) + requests: list[tuple[str, str, dict[str, str], float]] = [] + + def fake_urlopen(request, *, timeout): + requests.append( + ( + request.full_url, + request.get_method(), + {key.lower(): value for key, value in request.header_items()}, + timeout, + ) + ) + return _FakeHttpResponse(bodies.pop(0)) + + monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", fake_urlopen) + + assert module.password(args) == "scoped-runtime-password" + assert module.GCE_METADATA_EMAIL_URL == ( + "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" + ) + assert module.GCE_METADATA_TOKEN_URL == ( + "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" + ) + assert requests == [ + ( + module.GCE_METADATA_EMAIL_URL, + "GET", + {"metadata-flavor": "Google"}, + module.HTTP_TIMEOUT_SECONDS, + ), + ( + module.GCE_METADATA_TOKEN_URL, + "GET", + {"metadata-flavor": "Google"}, + module.HTTP_TIMEOUT_SECONDS, + ), + ( + module.RUNTIME_SECRET_ACCESS_URL, + "GET", + {"accept": "application/json", "authorization": "Bearer metadata-access-token"}, + module.HTTP_TIMEOUT_SECONDS, + ), + ] + assert capsys.readouterr() == ("", "") + + +def test_cloudsql_runtime_http_opener_disables_proxies_and_redirects() -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + proxy_handlers = [ + handler for handler in module.RUNTIME_HTTP_OPENER.handlers if isinstance(handler, module.urllib.request.ProxyHandler) + ] + redirect_handlers = [ + handler + for handler in module.RUNTIME_HTTP_OPENER.handlers + if isinstance(handler, module.urllib.request.HTTPRedirectHandler) + ] + + # An empty ProxyHandler suppresses build_opener's environment-backed default; + # urllib omits the empty handler itself from the final chain. + assert proxy_handlers == [] + assert module.RUNTIME_PROXY_HANDLER.proxies == {} + assert len(redirect_handlers) == 1 + assert type(redirect_handlers[0]) is module._RejectRedirectHandler + + +def test_cloudsql_runtime_rejects_wrong_metadata_service_account_before_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + calls: list[str] = [] + + def fake_urlopen(request, *, timeout): + assert timeout == module.HTTP_TIMEOUT_SECONDS + calls.append(request.full_url) + return _FakeHttpResponse(b"unexpected-sa@teleo-501523.iam.gserviceaccount.com") + + monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", fake_urlopen) + + with pytest.raises(SystemExit, match="did not match the required service account"): + module.password(args) + assert calls == [module.GCE_METADATA_EMAIL_URL] + + +@pytest.mark.parametrize( + "token_body", + [ + b"not-json", + b"[]", + b'{"access_token":"token","token_type":"Basic","expires_in":300}', + b'{"access_token":"token with spaces","token_type":"Bearer","expires_in":300}', + b'{"access_token":"token","token_type":"Bearer","expires_in":0}', + b'{"access_token":"one","access_token":"two","token_type":"Bearer","expires_in":300}', + ], +) +def test_cloudsql_runtime_rejects_malformed_metadata_tokens_without_leaking_them( + monkeypatch: pytest.MonkeyPatch, + token_body: bytes, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + bodies = [module.RUNTIME_SERVICE_ACCOUNT.encode("ascii"), token_body] + monkeypatch.setattr( + module.RUNTIME_HTTP_OPENER, + "open", + lambda _request, *, timeout: _FakeHttpResponse(bodies.pop(0)), + ) + + with pytest.raises(SystemExit, match="Runtime metadata token response was invalid") as captured: + module.password(args) + assert "token with spaces" not in str(captured.value) + + +@pytest.mark.parametrize( + "secret_body", + [ + b"not-json", + b"[]", + b"{}", + b'{"payload":{"data":"%%%"}}', + json.dumps({"payload": {"data": base64.b64encode(b"x" * 4097).decode("ascii")}}).encode(), + json.dumps({"payload": {"data": base64.b64encode(b"line1\nline2").decode("ascii")}}).encode(), + json.dumps({"payload": {"data": base64.b64encode(b"line1\rline2").decode("ascii")}}).encode(), + json.dumps({"payload": {"data": base64.b64encode(b"nul\x00byte").decode("ascii")}}).encode(), + ], +) +def test_cloudsql_runtime_rejects_malformed_secret_payloads_without_leaking_them( + monkeypatch: pytest.MonkeyPatch, + secret_body: bytes, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + bodies = _runtime_http_bodies(module) + bodies[-1] = secret_body + monkeypatch.setattr( + module.RUNTIME_HTTP_OPENER, + "open", + lambda _request, *, timeout: _FakeHttpResponse(bodies.pop(0)), + ) + + with pytest.raises(SystemExit, match=r"Scoped runtime secret .* invalid") as captured: + module.password(args) + assert secret_body.decode("utf-8", errors="ignore") not in str(captured.value) + + +def test_cloudsql_runtime_admin_secret_is_impossible_through_arguments( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module, password_secret="gcp-teleo-pgvector-standby-postgres-password") + _configure_runtime_environment(monkeypatch) + monkeypatch.setattr( + module.RUNTIME_HTTP_OPENER, + "open", + lambda *_args, **_kwargs: pytest.fail("metadata must not be queried for an admin secret argument"), + ) + + with pytest.raises(SystemExit, match="requires scoped password secret"): + module.password(args) + + +def test_cloudsql_runtime_http_and_psql_failures_never_echo_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + dummy = "DUMMY-RUNTIME-SECRET-DO-NOT-LEAK" + + def failed_urlopen(_request, *, timeout): + raise RuntimeError(f"provider echoed {dummy}") + + monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", failed_urlopen) + with pytest.raises(SystemExit) as http_error: + module.password(args) + assert dummy not in str(http_error.value) + + monkeypatch.setattr(module, "runtime_password_from_metadata", lambda: dummy) + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=2, stdout=dummy, stderr=dummy), + ) + with pytest.raises(SystemExit, match="Cloud SQL query failed with exit status 2") as psql_error: + module.run_psql(args, "select 1;") + assert dummy not in str(psql_error.value) + + +def test_cloudsql_runtime_read_marker_binds_database_user_private_server_and_ssl( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + marker = { + "database": module.RUNTIME_CLOUDSQL_DB, + "database_user": module.RUNTIME_DB_USER, + "server_addr": module.RUNTIME_CLOUDSQL_HOST, + "server_port": int(module.RUNTIME_CLOUDSQL_PORT), + "ssl": True, + "ssl_version": "TLSv1.3", + "system_identifier": "system-id", + "wal_lsn": "0/1", + } + captured: dict[str, object] = {} + + def fake_psql_json_lines(_args, sql, db=None): + captured["sql"] = sql + captured["db"] = db + return [marker] + + monkeypatch.setattr(module, "psql_json_lines", fake_psql_json_lines) + + assert module.database_read_marker(args) == marker + assert captured["db"] == module.RUNTIME_CLOUDSQL_DB + assert "inet_server_addr()" in str(captured["sql"]) + assert "pg_stat_ssl" in str(captured["sql"]) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("database", "teleo_kb", "unexpected database identity"), + ("database_user", "postgres", "unexpected database identity"), + ("server_addr", "127.0.0.1", "unexpected server address"), + ("server_port", 6543, "unexpected server port"), + ("ssl", False, "encrypted connection"), + ], +) +def test_cloudsql_runtime_read_marker_rejects_wrong_live_identity( + monkeypatch: pytest.MonkeyPatch, + field: str, + value: object, + message: str, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + marker = { + "database": module.RUNTIME_CLOUDSQL_DB, + "database_user": module.RUNTIME_DB_USER, + "server_addr": module.RUNTIME_CLOUDSQL_HOST, + "server_port": int(module.RUNTIME_CLOUDSQL_PORT), + "ssl": True, + "ssl_version": "TLSv1.3", + "system_identifier": "system-id", + "wal_lsn": "0/1", + } + marker[field] = value + monkeypatch.setattr(module, "psql_json_lines", lambda *_args, **_kwargs: [marker]) + + with pytest.raises(SystemExit, match=message): + module.database_read_marker(args) def test_cloudsql_proposal_calls_leave_leo_identity_to_the_database(monkeypatch: pytest.MonkeyPatch) -> None: @@ -390,6 +835,7 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None: assert "can_insert_proposals_directly" in sql assert "can_create_database_objects" in sql assert "can_create_temp_objects" in sql + assert "can_connect_legacy_database" in sql assert "unexpectedly has database CREATE" in sql assert "REVOKE TEMP FROM PUBLIC would be a" in sql assert "can execute unexpected SECURITY DEFINER functions" in sql @@ -407,6 +853,15 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None: ) assert "grant update" not in sql.lower() assert "grant delete" not in sql.lower() + assert "create role leoclean_kb_runtime nologin" in sql.lower() + disable_statement = "alter role leoclean_kb_runtime\n with nologin nosuperuser" + password_statement = "alter role leoclean_kb_runtime password" + assert sql.lower().index(disable_statement) < sql.lower().index(password_statement) + assert sql.lower().index(disable_statement) < sql.index("\\getenv runtime_password") + assert sql.index("\\getenv runtime_password") < sql.lower().index(password_statement) + assert sql.lower().index("with nologin nosuperuser") < sql.index("\\connect teleo_kb") + assert sql.index("\\connect teleo_kb") < sql.lower().rindex("with login nosuperuser") + assert "leoclean_kb_runtime final login attributes are unsafe" in sql def _load_module(path: Path): diff --git a/tests/test_teleo_agent_systemd.py b/tests/test_teleo_agent_systemd.py index 99c0535..f6edaaa 100644 --- a/tests/test_teleo_agent_systemd.py +++ b/tests/test_teleo_agent_systemd.py @@ -187,15 +187,60 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert 'DROPIN_NAME="10-cloudsql-memory.conf"' in script assert "20-canonical-kb.conf" not in script assert "sha256sum" in script + assert "--preflight-only" in script assert "--restart" in script - assert 'python3 "$tool_path" status --format json --redacted' in script + assert '/usr/bin/python3 "$tool_path" status --format json --redacted' in script assert "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" in script - assert "metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" in script - assert "env -u PGPASSWORD" in script - assert 'run_scoped_status pre "$remote_tmp/$TOOL_REL"' in script - assert 'run_scoped_status post "$profile_bin/cloudsql_memory_tool.py"' in script + assert "169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" in script + assert "ProxyHandler({})" in script + assert "env -i" in script + assert 'run_scoped_status pre "$payload_dir/$TOOL_REL"' in script + assert 'run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL"' in script + assert 'sudo -n -u teleo mkdir -p -- "$receipt_dir"' in script + assert 'if [ "$PREFLIGHT_ONLY" = true ]; then' in script + assert "run_service_wrapper_status post" in script + assert "run_service_wrapper_status verify" in script + assert '"$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted' in script + wrapper_probe = script.split("run_service_wrapper_status() {", maxsplit=1)[1].split( + "\n}\n\nservice_fingerprint", maxsplit=1 + )[0] + assert 'nsenter -t "$main_pid" -m --env' in wrapper_probe + assert "/usr/bin/setpriv" in wrapper_probe + assert "--reuid=teleo" in wrapper_probe + assert "TELEO_CLOUDSQL_" not in wrapper_probe + assert "/usr/bin/env -i" not in wrapper_probe + assert 'run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py"' in script + assert 'SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py"' in script + assert 'assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py"' in script + assert 'assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"' in script + assert "REMOTE_CLOUDSDK_CONFIG" not in script + assert "--property=DropInPaths" in script + assert "--property=EnvironmentFiles" in script + assert "Service has an effective EnvironmentFile" in script + assert "Service has a later drop-in" in script + assert "assert_administrator_secret_denied" in script + assert '"stdout_discarded": True' in script + assert '"credential_context": "systemd_main_pid_environment"' in script + administrator_probe = script.split("assert_administrator_secret_denied() {", maxsplit=1)[1].split( + "\n}\n\nrun_scoped_status", maxsplit=1 + )[0] + assert 'nsenter -t "$main_pid" -m --env' in administrator_probe + assert "/usr/bin/setpriv" in administrator_probe + assert "env -i" not in administrator_probe + assert 'preflight_service_before="$(service_fingerprint)"' in script + assert 'assert_service_fingerprint "$preflight_service_before"' in script assert "backup_path" in script + assert "backup_directory_state" in script assert "restore_path" in script + assert "restore_directory_state" in script + assert 'runtime_grandparent="$(dirname "$runtime_parent")"' in script + assert 'runtime_anchor="$(dirname "$runtime_grandparent")"' in script + assert 'dropin_parent="$(dirname "$dropin_dir")"' in script + assert 'backup_directory_state "$runtime_grandparent" runtime_grandparent' in script + assert 'restore_directory_state "$runtime_grandparent" runtime_grandparent' in script + assert 'sudo test -d "$runtime_anchor"' in script + assert 'sudo test -d "$dropin_parent"' in script + assert 'sudo rmdir -- "$target"' in script assert "rollback_runtime" in script assert "trap 'on_exit $?' EXIT" in script assert 'backup_path "$dropin_dir/$DROPIN_NAME" dropin' in script @@ -205,13 +250,303 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert 'sub_state" != "running"' in script assert 'sudo systemctl stop "$SERVICE"' in script assert 'sudo systemctl start "$SERVICE"' in script + assert 'REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"' in script + assert 'sudo install -d -o root -g root -m 0755 "$runtime_parent"' in script + assert 'sudo install -d -o root -g root -m 0755 "$runtime_bin"' in script + assert 'remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)"' in script + assert 'sudo install -d -o root -g root -m 0755 "$payload_dir"' in script + assert 'sudo install -d -o root -g root -m 0700 "$backup_dir"' in script + assert 'sudo install -d -o teleo -g teleo -m 0700 "$work_dir"' in script + assert "sudo tar --no-same-owner --no-same-permissions" in script + assert 'sudo chown -R root:root "$payload_dir"' in script + assert 'sudo find "$payload_dir" -type d -exec chmod 0555 {} +' in script + assert 'sudo find "$payload_dir" -type l -print -quit' in script + assert 'sudo -n -u teleo test ! -w "$payload_dir"' in script + assert 'sudo -n -u teleo test ! -w "$backup_dir"' in script + assert 'sudo touch "$backup_dir/$name.present"' in script + assert 'sudo test -f "$backup_dir/$name.present"' in script + assert 'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage"' in script + assert 'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage"' in script + assert ( + 'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" ' + '"$permission_verifier_stage"' in script + ) + assert ( + 'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" ' + '"$service_env_verifier_stage"' in script + ) + assert 'assert_installed_files\nsudo systemctl daemon-reload' in script + assert 'tar -C "$remote_tmp" -xf -' not in script + assert 'mkdir -p "$backup_dir"' not in script + assert 'for protected_file in \\\n "$wrapper" \\' in script + assert '"$permission_verifier" \\' in script + assert '"$service_env_verifier" \\' in script + assert 'sudo -n -u teleo test ! -w "$protected_file"' in script + assert 'for protected_dir in \\\n /usr \\' in script + assert "/usr/local/libexec/livingip" in script + assert 'sudo -n -u teleo test ! -w "$protected_dir"' in script + assert 'for executable_dir in \\\n /usr/local/sbin \\' in script + assert 'sudo stat -Lc \'%U:%G\' "$executable_dir"' in script + assert 'sudo -n -u teleo test ! -w "$executable_dir"' in script + assert 'for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3' in script + assert '/usr/bin/curl -q --noproxy' in script + assert 'test "$(sudo cat "$revision")" = "$SOURCE_REVISION"' in script + assert 'assert_existing_root_directory "$existing_directory"' in script + assert 'assert_existing_root_file "$existing_file"' in script + assert 'validate_restored_path "$runtime_bin/teleo-kb" wrapper' in script + assert 'validate_restored_directory "$runtime_bin" runtime_bin' in script + rollback = script.split("rollback_runtime() {", maxsplit=1)[1].split("\n}\n\non_exit()", maxsplit=1)[0] + assert rollback.index("validate_restored_path") < rollback.index("systemctl daemon-reload") + assert rollback.index("systemctl daemon-reload") < rollback.index('systemctl start "$SERVICE"') + assert rollback.count('if [ "$rollback_rc" -eq 0 ]; then') >= 3 + deploy_tail = script.split("mutation_started=true", maxsplit=1)[1] + assert deploy_tail.index("assert_installed_files") < deploy_tail.index("systemctl daemon-reload") + assert deploy_tail.index("systemctl daemon-reload") < deploy_tail.index("assert_unit_configuration") + assert deploy_tail.index("assert_unit_configuration") < deploy_tail.index('systemctl start "$SERVICE"') + for install_line in ( + 'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL"', + 'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL"', + 'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL"', + 'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL"', + 'sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL"', + ): + assert install_line in deploy_tail + assert 'sudo install -o root -g root -m 0755 "$work_dir/' not in script + assert 'nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb"' in script + assert 'findmnt -T "$SERVICE_PROFILE_BIN"' in script + assert 'findmnt -M "$SERVICE_HERMES_ROOT"' in script + assert 'findmnt -M "$SERVICE_PROFILE_ROOT/state"' in script + assert 'findmnt -M "$SERVICE_PROFILE_ROOT/workspace"' in script + assert "cap_sys_admin" in script + assert '"current_tier": "T3_live_readonly"' in script + assert '"status": "pass"' in script assert "UnsetEnvironment=PGPASSWORD" in dropin - assert "RuntimeDirectory=leoclean-gcp-prod-parallel" in dropin - assert "RuntimeDirectoryMode=0700" in dropin - assert "Environment=CLOUDSDK_CONFIG=/run/leoclean-gcp-prod-parallel" in dropin + assert "UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG" in dropin + assert "UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS" in dropin + assert "UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT" in dropin + assert "UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT" in dropin + assert "UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY" in dropin + assert "UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE" in dropin + assert ( + "BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin" in dropin + ) + assert "ReadOnlyPaths=/home/teleo/.hermes" in dropin + assert ( + "ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state " + "/home/teleo/.hermes/profiles/leoclean/workspace" in dropin + ) + assert "CapabilityBoundingSet=~CAP_SYS_ADMIN" in dropin + assert "RuntimeDirectory=" not in dropin + assert "Environment=CLOUDSDK_" not in dropin + assert "Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" in dropin assert "TELEO_KB_MODE=cloudsql" in dropin + assert "TELEO_GCP_METADATA_ONLY=1" in dropin assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in dropin assert "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" in dropin assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in dropin assert not any(line.startswith("Environment=PGPASSWORD") for line in dropin.splitlines()) assert "gcp-teleo-pgvector-standby-postgres-password" not in dropin + + +def test_gcp_runtime_embedded_remote_programs_are_bash_syntax_valid() -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + blocks: dict[str, str] = {} + + for variable in ("remote_helpers", "verify_body", "sync_body"): + marker = f"{variable}=$(cat <<'REMOTE'\n" + assert script.count(marker) == 1 + start = script.index(marker) + len(marker) + end = script.index("\nREMOTE\n)", start) + blocks[variable] = script[start:end] + + for name, body in ( + *blocks.items(), + ("composed_verify", blocks["remote_helpers"] + "\n" + blocks["verify_body"]), + ("composed_sync", blocks["remote_helpers"] + "\n" + blocks["sync_body"]), + ): + completed = subprocess.run( + ["bash", "-n"], + input=body, + capture_output=True, + check=False, + text=True, + ) + assert completed.returncode == 0, f"{name}: {completed.stderr}" + + embedded_python = [part.split("\nPY", maxsplit=1)[0] for part in blocks["remote_helpers"].split("<<'PY'\n")[1:]] + assert len(embedded_python) == 2 + for index, body in enumerate(embedded_python): + completed = subprocess.run( + ["python3", "-c", "import sys; compile(sys.stdin.read(), '', 'exec')"], + input=body, + capture_output=True, + check=False, + text=True, + ) + assert completed.returncode == 0, f"embedded_python_{index}: {completed.stderr}" + + +def test_gcp_runtime_rejects_preexisting_symlink_before_mutation(tmp_path: Path) -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("assert_existing_root_directory() {") + function_end = script.index("\n\nbackup_path() {", function_start) + trust_functions = script[function_start:function_end] + real_directory = tmp_path / "real-runtime" + real_directory.mkdir() + symlink = tmp_path / "runtime-link" + symlink.symlink_to(real_directory, target_is_directory=True) + harness = f""" +set -euo pipefail +sudo() {{ + if [ "${{1:-}}" = -n ]; then + shift + if [ "${{1:-}}" = -u ]; then shift 2; fi + fi + "$@" +}} +{trust_functions} +assert_existing_root_directory "$1" +""" + completed = subprocess.run( + ["bash", "-s", "--", str(symlink)], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode != 0 + + +def test_gcp_runtime_executable_configuration_override_gate() -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("assert_unit_configuration() {") + function_end = script.index("\n\nassert_runtime_environment() {", function_start) + unit_configuration_function = script[function_start:function_end] + harness = f""" +set -euo pipefail +SERVICE=leoclean.service +REMOTE_DROPIN_DIR=/etc/systemd/system/leoclean.service.d +DROPIN_NAME=10-cloudsql-memory.conf +systemctl() {{ + case "$*" in + *--property=DropInPaths*) printf '%s\\n' "${{FAKE_DROPINS:-}}" ;; + *--property=EnvironmentFiles*) printf '%s\\n' "${{FAKE_ENVIRONMENT_FILES:-}}" ;; + *) return 90 ;; + esac +}} +{unit_configuration_function} +assert_unit_configuration +""" + reviewed = "/etc/systemd/system/leoclean.service.d/10-cloudsql-memory.conf" + cases = ( + { + "FAKE_DROPINS": f"{reviewed} /etc/systemd/system/leoclean.service.d/99-conflict.conf", + "FAKE_ENVIRONMENT_FILES": "", + }, + {"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": "/etc/leoclean-runtime.env"}, + ) + for environment in cases: + completed = subprocess.run( + ["bash", "-s"], + input=harness, + capture_output=True, + check=False, + env=environment, + text=True, + ) + assert completed.returncode != 0 + + allowed = subprocess.run( + ["bash", "-s"], + input=harness, + capture_output=True, + check=False, + env={"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": ""}, + text=True, + ) + assert allowed.returncode == 0, allowed.stderr + + +def test_gcp_runtime_absence_rollback_removes_every_created_directory(tmp_path: Path) -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("backup_directory_state() {") + function_end = script.index("\nrestore_path() {", function_start) + directory_functions = script[function_start:function_end] + harness = f""" +set -euo pipefail +sudo() {{ "$@"; }} +root="$1" +backup_dir="$root/backup" +mkdir -p "$backup_dir" "$root/usr/local" "$root/etc/systemd/system" +runtime_grandparent="$root/usr/local/libexec" +runtime_parent="$runtime_grandparent/livingip" +runtime_bin="$runtime_parent/leoclean-kb" +dropin_dir="$root/etc/systemd/system/service.d" +{directory_functions} +backup_directory_state "$runtime_grandparent" runtime_grandparent +backup_directory_state "$runtime_parent" runtime_parent +backup_directory_state "$runtime_bin" runtime_bin +backup_directory_state "$dropin_dir" dropin_dir +mkdir -p "$runtime_bin" "$dropin_dir" +restore_directory_state "$runtime_bin" runtime_bin +restore_directory_state "$runtime_parent" runtime_parent +restore_directory_state "$runtime_grandparent" runtime_grandparent +restore_directory_state "$dropin_dir" dropin_dir +test ! -e "$runtime_grandparent" +test ! -e "$dropin_dir" +test -d "$root/usr/local" +test -d "$root/etc/systemd/system" +""" + completed = subprocess.run( + ["bash", "-s", "--", str(tmp_path)], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + + +def test_gcp_runtime_rollback_never_reloads_or_restarts_after_validation_failure(tmp_path: Path) -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("rollback_runtime() {") + function_end = script.index("\n\non_exit() {", function_start) + rollback_function = script[function_start:function_end] + action_log = tmp_path / "systemctl-actions" + harness = f""" +set -u +SERVICE=leoclean.service +runtime_bin=/usr/local/libexec/livingip/leoclean-kb +runtime_parent=/usr/local/libexec/livingip +runtime_grandparent=/usr/local/libexec +dropin_dir=/etc/systemd/system/leoclean.service.d +DROPIN_NAME=10-cloudsql-memory.conf +action_log="$1" +sudo() {{ + if [ "${{1:-}}" = systemctl ]; then + printf '%s\\n' "$2" >>"$action_log" + return 0 + fi + "$@" +}} +restore_path() {{ return 0; }} +restore_directory_state() {{ return 0; }} +validate_restored_path() {{ [ "$2" != wrapper ]; }} +validate_restored_directory() {{ return 0; }} +assert_unit_configuration() {{ printf '%s\\n' unit-configuration >>"$action_log"; }} +assert_service_running() {{ printf '%s\\n' assert-running >>"$action_log"; }} +{rollback_function} +rollback_runtime +""" + completed = subprocess.run( + ["bash", "-s", "--", str(action_log)], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode != 0 + assert action_log.read_text(encoding="utf-8").splitlines() == ["stop"] From 7b16c6d3cc0fc5284526b11f16878bec112eb3d0 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 07:16:40 +0200 Subject: [PATCH 3/7] fix: prove least-privilege GCP Leo runtime context --- deploy/sync-gcp-leoclean-runtime.sh | 681 ++++++++++++-- docs/gcp-leoclean-runtime-reconciliation.md | 111 ++- .../leoclean-bin/cloudsql_memory_tool.py | 94 +- hermes-agent/leoclean-bin/teleo-kb-gcp | 52 ++ ops/gcp-teleo-pgvector-standby-server-ca.pem | 21 + ops/gcp_leoclean_runtime_role.sql | 848 ++++++++++++++---- ops/provision_gcp_leoclean_runtime_role.sh | 150 ++++ ...verify_gcp_leoclean_runtime_permissions.py | 699 +++++++++++---- ...verify_gcp_leoclean_service_environment.py | 66 +- ...run_gcp_generated_db_direct_claim_suite.py | 29 +- .../run_gcp_generated_db_working_leo_suite.py | 28 +- .../leoclean-gcp-prod-parallel-cloudsql.conf | 25 +- ...est_gcp_generated_db_direct_claim_suite.py | 8 +- ...test_gcp_generated_db_working_leo_suite.py | 8 + .../test_gcp_leoclean_runtime_permissions.py | 95 +- .../test_gcp_leoclean_service_environment.py | 6 +- .../test_hermes_leoclean_kb_bridge_source.py | 104 ++- tests/test_teleo_agent_systemd.py | 351 +++++++- 18 files changed, 2840 insertions(+), 536 deletions(-) create mode 100755 hermes-agent/leoclean-bin/teleo-kb-gcp create mode 100644 ops/gcp-teleo-pgvector-standby-server-ca.pem create mode 100755 ops/provision_gcp_leoclean_runtime_role.sh diff --git a/deploy/sync-gcp-leoclean-runtime.sh b/deploy/sync-gcp-leoclean-runtime.sh index 25a7976..b0edacb 100755 --- a/deploy/sync-gcp-leoclean-runtime.sh +++ b/deploy/sync-gcp-leoclean-runtime.sh @@ -16,11 +16,14 @@ SERVICE_PROFILE_ROOT="${SERVICE_PROFILE_ROOT:-/home/teleo/.hermes/profiles/leocl SERVICE_HERMES_ROOT="${SERVICE_HERMES_ROOT:-/home/teleo/.hermes}" REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}" EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}" +EXPECTED_HERMES_EXECUTABLE="${EXPECTED_HERMES_EXECUTABLE:-/home/teleo/.hermes/hermes-agent/venv/bin/hermes}" +EXPECTED_HERMES_PYTHON="${EXPECTED_HERMES_PYTHON:-/home/teleo/.hermes/hermes-agent/venv/bin/python3}" -WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb" +WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb-gcp" TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py" PERMISSION_VERIFIER_REL="ops/verify_gcp_leoclean_runtime_permissions.py" SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py" +SERVER_CA_REL="ops/gcp-teleo-pgvector-standby-server-ca.pem" DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf" DROPIN_NAME="10-cloudsql-memory.conf" @@ -70,6 +73,7 @@ for path in \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ "$DROPIN_REL"; do if [ ! -f "$REPO_ROOT/$path" ]; then echo "Required source file is missing: $path" >&2 @@ -93,16 +97,37 @@ if ! grep -Fxq "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$RE echo "Runtime drop-in does not bind the root-controlled runtime into the service profile." >&2 exit 65 fi -if ! grep -Fxq "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$REPO_ROOT/$DROPIN_REL"; then - echo "Runtime drop-in does not protect the service-profile ancestor from replacement." >&2 +if ! grep -Fxq 'ReadOnlyPaths=/home/teleo' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not make the complete service home read-only." >&2 exit 65 fi if ! grep -Fxq "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in does not limit service-profile writes to state and workspace." >&2 exit 65 fi -if ! grep -Fxq 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$REPO_ROOT/$DROPIN_REL"; then - echo "Runtime drop-in must remove CAP_SYS_ADMIN from the service." >&2 +if ! grep -Fxq 'CapabilityBoundingSet=' "$REPO_ROOT/$DROPIN_REL" || + ! grep -Fxq 'AmbientCapabilities=' "$REPO_ROOT/$DROPIN_REL" || + ! grep -Fxq 'NoNewPrivileges=yes' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must remove all capabilities and enforce no-new-privileges." >&2 + exit 65 +fi +if ! grep -Fxq 'Group=teleo' "$REPO_ROOT/$DROPIN_REL" || + ! grep -Fxq 'SupplementaryGroups=teleo' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must bind the exact service group set." >&2 + exit 65 +fi +if ! grep -Fxq 'UnsetEnvironment=OPENSSL_CONF OPENSSL_MODULES OPENSSL_ENGINES GCONV_PATH CREDENTIALS_DIRECTORY' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must remove startup-loader and systemd-credential injection fields." >&2 + exit 65 +fi +if ! grep -Fxq 'InaccessiblePaths=-/home/teleo/.config/gcloud -/home/teleo/.pgpass -/home/teleo/.pg_service.conf -/home/teleo/.postgresql' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must mask default GCP and PostgreSQL credential paths." >&2 + exit 65 +fi +if ! grep -Fxq \ + 'UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT' \ + "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must unset optional Cloud SQL and memory controls." >&2 exit 65 fi @@ -112,6 +137,7 @@ if ! git -C "$REPO_ROOT" diff --quiet HEAD -- \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ "$DROPIN_REL"; then echo "Refusing to deploy runtime files that do not match HEAD." >&2 exit 65 @@ -121,6 +147,7 @@ if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ "$DROPIN_REL")" ]; then echo "Refusing to deploy untracked runtime files." >&2 exit 65 @@ -137,6 +164,7 @@ WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')" TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')" PERMISSION_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" SERVICE_ENV_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" +SERVER_CA_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVER_CA_REL" | awk '{print $1}')" DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')" GCP_SSH=( @@ -147,8 +175,22 @@ GCP_SSH=( ) remote_helpers=$(cat <<'REMOTE' +assert_exact_word_set() { + local actual="$1" expected="$2" label="$3" actual_sorted expected_sorted word + actual_sorted="$(for word in $actual; do printf '%s\n' "${word#-}"; done | sort)" + expected_sorted="$(for word in $expected; do printf '%s\n' "${word#-}"; done | sort)" + if [ "$actual_sorted" != "$expected_sorted" ]; then + echo "Service effective $label differs from the reviewed exact allowlist." >&2 + return 1 + fi +} + assert_service_running() { - local active_state sub_state main_pid unit_user process_user + local require_reviewed="${1:-true}" + local active_state sub_state main_pid unit_user process_user teleo_uid teleo_gid + local uid_fields gid_fields groups_field exec_start process_executable + local seccomp_mode seccomp_filters lsm_context + local host_user_namespace service_user_namespace active_state="$(systemctl show "$SERVICE" --property=ActiveState --value)" sub_state="$(systemctl show "$SERVICE" --property=SubState --value)" if [ "$active_state" != "active" ] || [ "$sub_state" != "running" ]; then @@ -173,17 +215,91 @@ assert_service_running() { echo "Service process user is not teleo: $process_user" >&2 return 1 fi + if [ "$require_reviewed" = true ]; then + teleo_uid="$(id -u teleo)" + teleo_gid="$(id -g teleo)" + uid_fields="$(awk '/^Uid:/{print $2 ":" $3 ":" $4 ":" $5}' "/proc/$main_pid/status")" + gid_fields="$(awk '/^Gid:/{print $2 ":" $3 ":" $4 ":" $5}' "/proc/$main_pid/status")" + groups_field="$(awk '/^Groups:/{for (i=2;i<=NF;i++) printf "%s%s", (i==2 ? "" : ":"), $i}' "/proc/$main_pid/status")" + if [ "$uid_fields" != "$teleo_uid:$teleo_uid:$teleo_uid:$teleo_uid" ] || + [ "$gid_fields" != "$teleo_gid:$teleo_gid:$teleo_gid:$teleo_gid" ] || + [ "$groups_field" != "$teleo_gid" ]; then + echo "Service process does not have the exact reviewed uid/gid/group set." >&2 + return 1 + fi + fi + exec_start="$(systemctl show "$SERVICE" --property=ExecStart --value)" + case "$exec_start" in + *"path=$EXPECTED_HERMES_EXECUTABLE ; argv[]=$EXPECTED_HERMES_EXECUTABLE -p leoclean gateway run ;"*) ;; + *) + echo "Service effective ExecStart is not the reviewed leoclean Hermes gateway." >&2 + return 1 + ;; + esac + sudo /usr/bin/python3 -I - "$main_pid" "$EXPECTED_HERMES_PYTHON" "$EXPECTED_HERMES_EXECUTABLE" <<'PY' +import os +import sys +from pathlib import Path + +pid, expected_python, expected_hermes = sys.argv[1:] +raw = Path(f"/proc/{pid}/cmdline").read_bytes() +if not raw.endswith(b"\0") or b"\0\0" in raw: + raise SystemExit("service cmdline is malformed") +try: + argv = [part.decode("utf-8", "strict") for part in raw[:-1].split(b"\0")] +except UnicodeDecodeError: + raise SystemExit("service cmdline is not valid UTF-8") from None +if argv != [expected_python, expected_hermes, "-p", "leoclean", "gateway", "run"]: + raise SystemExit("service cmdline is not the reviewed leoclean Hermes gateway") +if os.path.realpath(f"/proc/{pid}/exe") != os.path.realpath(expected_python): + raise SystemExit("service executable is not the reviewed Hermes interpreter") +PY + if [ "$require_reviewed" = true ]; then + for capability_field in CapInh CapPrm CapEff CapBnd CapAmb; do + if [ "$(awk -v key="$capability_field:" '$1 == key {print $2}' "/proc/$main_pid/status")" != 0000000000000000 ]; then + echo "Service process retains capabilities in $capability_field." >&2 + return 1 + fi + done + if [ "$(awk '/^NoNewPrivs:/{print $2}' "/proc/$main_pid/status")" != 1 ]; then + echo "Service process does not enforce no-new-privileges." >&2 + return 1 + fi + seccomp_mode="$(awk '/^Seccomp:/{print $2}' "/proc/$main_pid/status")" + seccomp_filters="$(awk '/^Seccomp_filters:/{print $2}' "/proc/$main_pid/status")" + lsm_context="$(sudo cat "/proc/$main_pid/attr/current")" + host_user_namespace="$(sudo stat -Lc '%d:%i' /proc/1/ns/user)" + service_user_namespace="$(sudo stat -Lc '%d:%i' "/proc/$main_pid/ns/user")" + if [ "$seccomp_mode" != 0 ] || + { [ -n "$seccomp_filters" ] && [ "$seccomp_filters" != 0 ]; } || + [ "$lsm_context" != unconfined ] || + [ "$service_user_namespace" != "$host_user_namespace" ]; then + echo "Service process has an unreviewed live seccomp, LSM, or user-namespace restriction." >&2 + return 1 + fi + fi } assert_unit_configuration() { local require_reviewed="${1:-true}" local dropin_path dropin_name dropin_paths environment_files reviewed_dropin_seen=false + local property value capability_set ambient_capabilities no_new_privileges unit_group supplementary_groups + local inaccessible_paths credential_path + local read_only_paths read_write_paths bind_paths bind_read_only_paths temporary_file_systems + local private_network private_users network_property dropin_paths="$(systemctl show "$SERVICE" --property=DropInPaths --value)" environment_files="$(systemctl show "$SERVICE" --property=EnvironmentFiles --value)" if [ -n "$environment_files" ]; then echo "Service has an effective EnvironmentFile, which is not allowed for the scoped runtime." >&2 return 1 fi + for property in LoadCredential LoadCredentialEncrypted SetCredential SetCredentialEncrypted ImportCredential ExecCondition ExecStartPre ExecStartPost ExecReload ExecStop ExecStopPost; do + value="$(systemctl show "$SERVICE" --property="$property" --value)" + if [ -n "$value" ]; then + echo "Service has an effective $property credential source." >&2 + return 1 + fi + done for dropin_path in $dropin_paths; do if [ "$dropin_path" = "$REMOTE_DROPIN_DIR/$DROPIN_NAME" ]; then reviewed_dropin_seen=true @@ -199,13 +315,82 @@ assert_unit_configuration() { echo "Service does not load the reviewed Cloud SQL drop-in." >&2 return 1 fi + if [ "$require_reviewed" = true ]; then + capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)" + ambient_capabilities="$(systemctl show "$SERVICE" --property=AmbientCapabilities --value)" + no_new_privileges="$(systemctl show "$SERVICE" --property=NoNewPrivileges --value)" + unit_group="$(systemctl show "$SERVICE" --property=Group --value)" + supplementary_groups="$(systemctl show "$SERVICE" --property=SupplementaryGroups --value)" + inaccessible_paths="$(systemctl show "$SERVICE" --property=InaccessiblePaths --value)" + read_only_paths="$(systemctl show "$SERVICE" --property=ReadOnlyPaths --value)" + read_write_paths="$(systemctl show "$SERVICE" --property=ReadWritePaths --value)" + bind_paths="$(systemctl show "$SERVICE" --property=BindPaths --value)" + bind_read_only_paths="$(systemctl show "$SERVICE" --property=BindReadOnlyPaths --value)" + temporary_file_systems="$(systemctl show "$SERVICE" --property=TemporaryFileSystem --value)" + if [ -n "$capability_set" ] || [ -n "$ambient_capabilities" ] || [ "$no_new_privileges" != yes ] || + [ "$unit_group" != teleo ] || [ "$supplementary_groups" != teleo ]; then + echo "Service effective capability, privilege, or group contract differs from the reviewed drop-in." >&2 + return 1 + fi + assert_exact_word_set "$read_only_paths" "/home/teleo" ReadOnlyPaths + assert_exact_word_set "$read_write_paths" "$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" ReadWritePaths + assert_exact_word_set "$bind_paths" "" BindPaths + assert_exact_word_set "$bind_read_only_paths" "$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" BindReadOnlyPaths + assert_exact_word_set "$temporary_file_systems" "" TemporaryFileSystem + assert_exact_word_set "$supplementary_groups" "teleo" SupplementaryGroups + assert_exact_word_set "$inaccessible_paths" "/home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql" InaccessiblePaths + private_network="$(systemctl show "$SERVICE" --property=PrivateNetwork --value)" + private_users="$(systemctl show "$SERVICE" --property=PrivateUsers --value)" + if [ "$private_network" != no ] || [ "$private_users" != no ]; then + echo "Service effective private network or user namespace differs from the reviewed host contract." >&2 + return 1 + fi + for network_property in \ + NetworkNamespacePath \ + IPAddressDeny \ + IPAddressAllow \ + RestrictAddressFamilies \ + RestrictNetworkInterfaces \ + SocketBindAllow \ + SocketBindDeny \ + SystemCallFilter \ + AppArmorProfile \ + SELinuxContext; do + value="$(systemctl show "$SERVICE" --property="$network_property" --value)" + if [ -n "$value" ]; then + echo "Service effective $network_property differs from the reviewed unrestricted transport contract." >&2 + return 1 + fi + done + for credential_path in /home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql; do + case " $inaccessible_paths " in + *" $credential_path "*) ;; + *) + echo "Service does not mask a default credential path: $credential_path" >&2 + return 1 + ;; + esac + done + fi } assert_runtime_environment() { local verifier_path="$1" require_reviewed="${2:-true}" main_pid main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" - sudo /usr/bin/python3 "$verifier_path" --pid "$main_pid" + if [ "$require_reviewed" = true ]; then + sudo /usr/bin/python3 -I "$verifier_path" --pid "$main_pid" + else + sudo /usr/bin/python3 -I "$verifier_path" --pid "$main_pid" --allow-missing-runtime-security-fields + fi assert_unit_configuration "$require_reviewed" + if [ "$require_reviewed" = true ]; then + sudo nsenter -t "$main_pid" -m -- test ! -e "/run/credentials/$SERVICE" + for credential_path in /home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql; do + sudo nsenter -t "$main_pid" -m -- \ + /usr/bin/setpriv --reuid=teleo --regid=teleo --clear-groups --no-new-privs \ + /usr/bin/test ! -r "$credential_path" + done + fi } assert_attached_service_account() { @@ -225,17 +410,240 @@ assert_attached_service_account() { echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email" } +run_in_service_cgroup() { + local main_pid="$1" control_group + shift + control_group="$(systemctl show "$SERVICE" --property=ControlGroup --value)" + if [ -z "$control_group" ]; then + echo "Service has no effective cgroup." >&2 + return 1 + fi + sudo /usr/bin/python3 -I - "$main_pid" "$control_group" "$@" <<'PY' +import ctypes +import os +import resource +import signal +import stat +import sys +from pathlib import Path, PurePosixPath + + +HANDLED_SIGNALS = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP) +PR_SET_PDEATHSIG = 1 + + +class ParentInterrupted(BaseException): + def __init__(self, signum: int) -> None: + self.signum = signum + + +def fail() -> None: + raise SystemExit("service-context runner failed closed") + + +def process_cgroup(pid: str) -> str: + try: + lines = Path(f"/proc/{pid}/cgroup").read_text(encoding="ascii").splitlines() + except (OSError, UnicodeError): + fail() + matches = [] + for line in lines: + parts = line.split(":", 2) + if len(parts) == 3 and parts[0] == "0" and parts[1] == "": + matches.append(parts[2]) + if len(matches) != 1: + fail() + value = matches[0] + path = PurePosixPath(value) + if not value.startswith("/") or ".." in path.parts or str(path) != value: + fail() + return value + + +def set_parent_death_signal(expected_parent: int) -> None: + libc = ctypes.CDLL(None, use_errno=True) + prctl = libc.prctl + prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] + prctl.restype = ctypes.c_int + if prctl(PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) != 0: + os._exit(125) + if os.getppid() != expected_parent: + os._exit(125) + + +def kill_and_reap(child: int) -> None: + try: + os.kill(child, signal.SIGKILL) + except ProcessLookupError: + pass + while True: + try: + os.waitpid(child, 0) + return + except InterruptedError: + continue + except ChildProcessError: + return + + +def supported_resource_ids() -> tuple[int, ...]: + identifiers = set() + for name in dir(resource): + if not name.startswith("RLIMIT_") or name == "RLIMIT_NLIMITS": + continue + identifier = getattr(resource, name) + if not isinstance(identifier, int) or identifier < 0: + continue + try: + resource.prlimit(os.getpid(), identifier) + except (OSError, ValueError): + continue + identifiers.add(identifier) + return tuple(sorted(identifiers)) + + +def copy_process_limits(source_pid: int, child: int) -> None: + snapshot = { + identifier: resource.prlimit(source_pid, identifier) + for identifier in supported_resource_ids() + } + for identifier, limits in snapshot.items(): + resource.prlimit(child, identifier, limits) + if resource.prlimit(child, identifier) != limits: + fail() + if any(resource.prlimit(source_pid, identifier) != limits for identifier, limits in snapshot.items()): + fail() + + +def no_op_stopped_child(_child: int) -> None: + return None + + +def run_stopped_child( + command: list[str], + expected_cgroup: str, + *, + move_child, + read_child_cgroup, + prepare_child=set_parent_death_signal, + configure_stopped_child=no_op_stopped_child, +) -> int: + if not command or not command[0].startswith("/"): + fail() + original_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + parent_pid = os.getpid() + try: + child = os.fork() + except BaseException: + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + raise + if child == 0: + try: + for signum in HANDLED_SIGNALS: + signal.signal(signum, signal.SIG_DFL) + prepare_child(parent_pid) + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + if os.getppid() != parent_pid: + os._exit(125) + os.kill(os.getpid(), signal.SIGSTOP) + os.execv(command[0], command) + except BaseException: + os._exit(126) + + previous_handlers = {} + reaped = False + interrupted = None + status = None + + def interrupt(signum, _frame) -> None: + signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + raise ParentInterrupted(signum) + + try: + for signum in HANDLED_SIGNALS: + previous_handlers[signum] = signal.signal(signum, interrupt) + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + waited, status = os.waitpid(child, os.WUNTRACED) + if waited != child or not os.WIFSTOPPED(status) or os.WSTOPSIG(status) != signal.SIGSTOP: + fail() + move_child(child) + if read_child_cgroup(str(child)) != expected_cgroup: + fail() + configure_stopped_child(child) + os.kill(child, signal.SIGCONT) + waited, status = os.waitpid(child, 0) + if waited != child: + fail() + reaped = True + except ParentInterrupted as exc: + interrupted = exc.signum + finally: + signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + if not reaped: + kill_and_reap(child) + for signum, previous in previous_handlers.items(): + signal.signal(signum, previous) + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + + if interrupted is not None: + signal.signal(interrupted, signal.SIG_DFL) + os.kill(os.getpid(), interrupted) + fail() + if status is None: + fail() + if os.WIFEXITED(status): + return os.WEXITSTATUS(status) + if os.WIFSIGNALED(status): + return 128 + os.WTERMSIG(status) + fail() + + +def main() -> int: + if len(sys.argv) < 4: + fail() + main_pid, expected_cgroup, *command = sys.argv[1:] + if not main_pid.isascii() or not main_pid.isdecimal() or int(main_pid) <= 0: + fail() + if process_cgroup(main_pid) != expected_cgroup: + fail() + cgroup_root = Path("/sys/fs/cgroup").resolve(strict=True) + cgroup_dir = (cgroup_root / expected_cgroup.lstrip("/")).resolve(strict=True) + try: + cgroup_dir.relative_to(cgroup_root) + except ValueError: + fail() + cgroup_procs = cgroup_dir / "cgroup.procs" + try: + if not stat.S_ISREG(cgroup_procs.stat().st_mode): + fail() + except OSError: + fail() + + def move_child(child: int) -> None: + with cgroup_procs.open("w", encoding="ascii") as handle: + handle.write(f"{child}\n") + + return run_stopped_child( + command, + expected_cgroup, + move_child=move_child, + read_child_cgroup=process_cgroup, + configure_stopped_child=lambda child: copy_process_limits(int(main_pid), child), + ) + + +if __name__ == "__main__": + raise SystemExit(main()) +PY +} + assert_administrator_secret_denied() { - local main_pid + local main_pid probe_program main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" - sudo nsenter -t "$main_pid" -m --env -- \ - /usr/bin/setpriv \ - --reuid=teleo \ - --regid=teleo \ - --init-groups \ - --no-new-privs \ - /usr/bin/python3 - "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" <<'PY' + probe_program="$(cat <<'PY' import json +import os +import subprocess import sys import urllib.error import urllib.parse @@ -280,20 +688,69 @@ try: else: response.close() raise RuntimeError + gcloud_command = [ + "/usr/bin/gcloud", + "secrets", + "versions", + "access", + "latest", + "--secret=gcp-teleo-pgvector-standby-postgres-password", + f"--project={project}", + "--quiet", + ] + default_discovery_environment = os.environ.copy() + for key in ( + "CLOUDSDK_CONFIG", + "GOOGLE_APPLICATION_CREDENTIALS", + "PGPASSFILE", + "PGSERVICE", + "XDG_CONFIG_HOME", + ): + default_discovery_environment.pop(key, None) + default_discovery_environment["HOME"] = "/home/teleo" + for environment in (os.environ.copy(), default_discovery_environment): + denied = subprocess.run( + gcloud_command, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + env=environment, + timeout=30, + check=False, + ) + denial_error = denied.stderr[:65536] + if denied.returncode == 0: + raise RuntimeError + if "PERMISSION_DENIED" not in denial_error or "secretmanager.versions.access" not in denial_error: + raise RuntimeError except Exception: raise SystemExit("administrator secret was not the exact expected metadata-identity IAM denial") from None print(json.dumps({ "administrator_secret_access": "iam_permission_denied", + "ambient_gcloud_administrator_secret_access": "iam_permission_denied", "credential_context": "systemd_main_pid_environment", - "identity_source": "gce_metadata", + "direct_metadata_administrator_secret_access": "iam_permission_denied", + "default_discovery_administrator_secret_access": "iam_permission_denied", + "identity_source": "gce_metadata_ambient_gcloud_and_default_discovery", "stdout_discarded": True, }, sort_keys=True)) PY +)" + run_in_service_cgroup "$main_pid" \ + /usr/bin/nsenter -t "$main_pid" -m -n --env -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --clear-groups \ + --no-new-privs \ + /usr/bin/python3 -I -c "$probe_program" "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" } run_scoped_status() { - local phase="$1" tool_path="$2" + local phase="$1" tool_path="$2" server_ca_path="${3:-$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem}" + local cloudsdk_config="${4:-$REMOTE_RUNTIME_BIN/gcloud-config}" assert_attached_service_account echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical" sudo -n -u teleo env -i \ @@ -308,7 +765,12 @@ run_scoped_status() { TELEO_CLOUDSQL_USER=leoclean_kb_runtime \ TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \ TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \ - /usr/bin/python3 "$tool_path" status --format json --redacted + CLOUDSDK_CONFIG="$cloudsdk_config" \ + PGSSLMODE=verify-ca \ + PGSSLROOTCERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ + /usr/bin/python3 -I "$tool_path" status --format json --redacted } run_service_wrapper_status() { @@ -318,11 +780,12 @@ run_service_wrapper_status() { echo "SCOPED_WRAPPER_STATUS phase=$phase user=teleo database=teleo_canonical" # --env imports the target MainPID's effective /proc environment. setpriv # drops back to the service's Unix identity without sudo resetting that env. - sudo nsenter -t "$main_pid" -m --env -- \ + run_in_service_cgroup "$main_pid" \ + /usr/bin/nsenter -t "$main_pid" -m -n --env -- \ /usr/bin/setpriv \ --reuid=teleo \ --regid=teleo \ - --init-groups \ + --clear-groups \ --no-new-privs \ "$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted } @@ -346,22 +809,44 @@ assert_service_fingerprint() { } run_permission_verifier() { - local phase="$1" verifier_path="$2" receipt_path="$3" receipt_dir run_id receipt_state + local phase="$1" verifier_path="$2" receipt_path="$3" + local server_ca_path="${4:-$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem}" receipt_dir run_id receipt_state + local main_pid + local cloudsdk_config="${5:-$REMOTE_RUNTIME_BIN/gcloud-config}" run_id="$(date -u +%Y%m%d%H%M%S)-$phase" receipt_dir="$(dirname "$receipt_path")" sudo -n -u teleo mkdir -p -- "$receipt_dir" sudo -n -u teleo chmod 0700 "$receipt_dir" echo "NEGATIVE_PERMISSION_RECEIPT phase=$phase run_id=$run_id" - sudo -n -u teleo env -i \ - HOME=/home/teleo \ - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - /usr/bin/python3 "$verifier_path" --run-id "$run_id" --output "$receipt_path" + if [ "$phase" = pre ]; then + sudo -n -u teleo env -i \ + HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ + /usr/bin/python3 -I "$verifier_path" --run-id "$run_id" --output "$receipt_path" + else + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + run_in_service_cgroup "$main_pid" \ + /usr/bin/nsenter -t "$main_pid" -m -n -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --clear-groups \ + --no-new-privs \ + /usr/bin/env -i \ + HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ + /usr/bin/python3 -I "$verifier_path" --run-id "$run_id" --output "$receipt_path" + fi receipt_state="$(sudo stat -c '%U:%G:%a' "$receipt_path")" if [ "$receipt_state" != "teleo:teleo:600" ]; then echo "Permission receipt has unexpected ownership or mode: $receipt_state" >&2 return 1 fi - sudo /usr/bin/python3 - "$receipt_path" <<'PY' + sudo /usr/bin/python3 -I - "$receipt_path" <<'PY' import json import sys from pathlib import Path @@ -381,33 +866,74 @@ print(json.dumps({"permission_receipt": "validated", "validated_fields": sorted( PY } +assert_trusted_executable() { + local requested="$1" resolved current owner + sudo test -x "$requested" + owner="$(sudo stat -c '%U' "$requested")" + if [ "$owner" != root ]; then + echo "Runtime executable path is not root-owned: $requested" >&2 + return 1 + fi + resolved="$(sudo readlink -f -- "$requested")" + sudo test -f "$resolved" + sudo test -x "$resolved" + current="$(dirname "$requested")" + while :; do + test "$(sudo stat -Lc '%U' "$current")" = root + sudo -n -u teleo test ! -w "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done + current="$resolved" + while :; do + test "$(sudo stat -Lc '%U' "$current")" = root + sudo -n -u teleo test ! -w "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done +} + assert_installed_files() { - local wrapper tool permission_verifier service_env_verifier dropin revision executable_dir protected_dir protected_file + local wrapper tool permission_verifier service_env_verifier server_ca gcloud_config dropin revision + local executable_dir protected_dir protected_file actual_entries expected_entries wrapper="$REMOTE_RUNTIME_BIN/teleo-kb" tool="$REMOTE_RUNTIME_BIN/cloudsql_memory_tool.py" permission_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" service_env_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" + server_ca="$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem" + gcloud_config="$REMOTE_RUNTIME_BIN/gcloud-config" dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME" revision="$REMOTE_RUNTIME_BIN/.livingip-runtime-revision" test "$(sudo sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA" test "$(sudo sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA" test "$(sudo sha256sum "$permission_verifier" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" test "$(sudo sha256sum "$service_env_verifier" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" + test "$(sudo sha256sum "$server_ca" | awk '{print $1}')" = "$SERVER_CA_SHA" test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA" test "$(sudo stat -c '%U:%G:%a' "$REMOTE_RUNTIME_BIN")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$wrapper")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$tool")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$permission_verifier")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$service_env_verifier")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$server_ca")" = "root:root:644" + test "$(sudo stat -c '%U:%G:%a' "$gcloud_config")" = "root:root:555" test "$(sudo stat -c '%U:%G:%a' "$revision")" = "root:root:644" test "$(sudo stat -c '%U:%G:%a' "$dropin")" = "root:root:644" test "$(sudo cat "$revision")" = "$SOURCE_REVISION" + sudo test -z "$(sudo find "$gcloud_config" -mindepth 1 -print -quit)" + expected_entries="$(printf '%s\n' .livingip-runtime-revision cloudsql-server-ca.pem cloudsql_memory_tool.py gcloud-config teleo-kb verify_gcp_leoclean_runtime_permissions.py verify_gcp_leoclean_service_environment.py | sort)" + actual_entries="$(sudo find "$REMOTE_RUNTIME_BIN" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" + if [ "$actual_entries" != "$expected_entries" ]; then + echo "Runtime directory contains an unreviewed or missing entry." >&2 + return 1 + fi for protected_dir in \ /usr \ /usr/local \ /usr/local/libexec \ /usr/local/libexec/livingip \ "$REMOTE_RUNTIME_BIN" \ + "$gcloud_config" \ "$REMOTE_DROPIN_DIR"; do sudo test -d "$protected_dir" sudo test ! -L "$protected_dir" @@ -419,6 +945,7 @@ assert_installed_files() { "$tool" \ "$permission_verifier" \ "$service_env_verifier" \ + "$server_ca" \ "$revision" \ "$dropin"; do sudo test -f "$protected_file" @@ -436,22 +963,27 @@ assert_installed_files() { test "$(sudo stat -Lc '%U:%G' "$executable_dir")" = "root:root" sudo -n -u teleo test ! -w "$executable_dir" done - for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3; do - sudo test -x "$protected_file" - test "$(sudo stat -Lc '%U:%G' "$protected_file")" = "root:root" - sudo -n -u teleo test ! -w "$protected_file" + for protected_file in /bin/bash /usr/bin/curl /usr/bin/env /usr/bin/gcloud /usr/bin/psql /usr/bin/python3 /usr/bin/setpriv; do + assert_trusted_executable "$protected_file" done sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin" sudo grep -E '^Environment=TELEO_(KB_MODE|GCP_PROJECT|GCP_METADATA_ONLY|CLOUDSQL_|CANONICAL_CLOUDSQL_DB)' "$dropin" - sudo grep -Fx "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$dropin" + sudo grep -Fx 'ReadOnlyPaths=/home/teleo' "$dropin" sudo grep -Fx "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$dropin" sudo grep -Fx "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$dropin" - sudo grep -Fx 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$dropin" + sudo grep -Fx 'CapabilityBoundingSet=' "$dropin" + sudo grep -Fx 'AmbientCapabilities=' "$dropin" + sudo grep -Fx 'NoNewPrivileges=yes' "$dropin" + sudo grep -Fx 'Group=teleo' "$dropin" + sudo grep -Fx 'SupplementaryGroups=teleo' "$dropin" + sudo grep -Fx \ + 'UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT' \ + "$dropin" } assert_deployed_files() { local main_pid namespace_state source_state mount_state - local hermes_mount_state state_mount_state workspace_mount_state capability_set + local home_mount_state state_mount_state workspace_mount_state capability_set ambient_capabilities assert_installed_files main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" @@ -469,13 +1001,13 @@ assert_deployed_files() { return 1 ;; esac - hermes_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_HERMES_ROOT" -n -o OPTIONS)" + home_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M /home/teleo -n -o OPTIONS)" state_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/state" -n -o OPTIONS)" workspace_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/workspace" -n -o OPTIONS)" - case ",$hermes_mount_state," in + case ",$home_mount_state," in *,ro,*) ;; *) - echo "Service .hermes ancestor mount is not read-only." >&2 + echo "Service home mount is not read-only." >&2 return 1 ;; esac @@ -501,37 +1033,42 @@ assert_deployed_files() { sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/state" sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/workspace" capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)" - if printf '%s\n' "$capability_set" | grep -Eqi '(^|[[:space:]])cap_sys_admin([[:space:]]|$)'; then - echo "Service capability bounding set still contains CAP_SYS_ADMIN." >&2 + ambient_capabilities="$(systemctl show "$SERVICE" --property=AmbientCapabilities --value)" + if [ -n "$capability_set" ] || [ -n "$ambient_capabilities" ]; then + echo "Service effective capability sets are not empty." >&2 return 1 fi test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb" | awk '{print $1}')" = "$WRAPPER_SHA" test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql_memory_tool.py" | awk '{print $1}')" = "$TOOL_SHA" test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_runtime_permissions.py" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_service_environment.py" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql-server-ca.pem" | awk '{print $1}')" = "$SERVER_CA_SHA" } REMOTE ) printf -v verify_context \ - 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \ - "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nEXPECTED_HERMES_EXECUTABLE=%q\nEXPECTED_HERMES_PYTHON=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nSERVER_CA_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$EXPECTED_HERMES_EXECUTABLE" "$EXPECTED_HERMES_PYTHON" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ - "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" "$SOURCE_REVISION" + "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$SERVER_CA_SHA" "$DROPIN_SHA" "$SOURCE_REVISION" verify_body=$(cat <<'REMOTE' set -euo pipefail -verification_tmp="$(mktemp -d /tmp/teleo-gcp-runtime-verify.XXXXXX)" +verification_tmp="$(sudo mktemp -d /run/teleo-gcp-runtime-verify.XXXXXX)" sudo chown teleo:teleo "$verification_tmp" sudo chmod 0700 "$verification_tmp" cleanup_verification() { sudo rm -rf -- "$verification_tmp"; } trap cleanup_verification EXIT assert_deployed_files assert_service_running +verification_service_before="$(service_fingerprint)" assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" run_service_wrapper_status verify assert_administrator_secret_denied run_permission_verifier verify "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" "$verification_tmp/permission-receipt.json" +assert_service_fingerprint "$verification_service_before" +assert_service_running systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager REMOTE ) @@ -539,7 +1076,7 @@ verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}" if $DRY_RUN; then echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE preflight_only=$PREFLIGHT_ONLY restart=$RESTART" - echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA dropin_sha256=$DROPIN_SHA" + echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA server_ca_sha256=$SERVER_CA_SHA dropin_sha256=$DROPIN_SHA" exit 0 fi @@ -554,15 +1091,15 @@ if ! $RESTART && ! $PREFLIGHT_ONLY; then fi printf -v sync_context \ - 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \ - "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nEXPECTED_HERMES_EXECUTABLE=%q\nEXPECTED_HERMES_PYTHON=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nSERVER_CA_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nSERVER_CA_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$EXPECTED_HERMES_EXECUTABLE" "$EXPECTED_HERMES_PYTHON" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ - "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" \ - "$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY" + "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$SERVER_CA_SHA" "$DROPIN_SHA" \ + "$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$SERVER_CA_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY" sync_body=$(cat <<'REMOTE' set -euo pipefail -remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)" +remote_tmp="$(sudo mktemp -d /run/teleo-gcp-leoclean-runtime.XXXXXX)" runtime_bin="$REMOTE_RUNTIME_BIN" runtime_parent="$(dirname "$runtime_bin")" runtime_grandparent="$(dirname "$runtime_parent")" @@ -576,6 +1113,7 @@ wrapper_stage="$runtime_bin/.teleo-kb.livingip-new.$$" tool_stage="$runtime_bin/.cloudsql-memory-tool.livingip-new.$$" permission_verifier_stage="$runtime_bin/.runtime-permission-verifier.livingip-new.$$" service_env_verifier_stage="$runtime_bin/.service-environment-verifier.livingip-new.$$" +server_ca_stage="$runtime_bin/.cloudsql-server-ca.livingip-new.$$" revision_stage="$runtime_bin/.livingip-runtime-revision.livingip-new.$$" dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$" mutation_started=false @@ -679,8 +1217,10 @@ rollback_runtime() { restore_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 restore_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 restore_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 + restore_path "$runtime_bin/cloudsql-server-ca.pem" server_ca || rollback_rc=1 restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 restore_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 + restore_directory_state "$runtime_bin/gcloud-config" gcloud_config || rollback_rc=1 restore_directory_state "$runtime_bin" runtime_bin || rollback_rc=1 restore_directory_state "$runtime_parent" runtime_parent || rollback_rc=1 restore_directory_state "$runtime_grandparent" runtime_grandparent || rollback_rc=1 @@ -690,11 +1230,13 @@ rollback_runtime() { validate_restored_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 validate_restored_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 validate_restored_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 + validate_restored_path "$runtime_bin/cloudsql-server-ca.pem" server_ca || rollback_rc=1 validate_restored_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 validate_restored_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 validate_restored_directory "$runtime_grandparent" runtime_grandparent || rollback_rc=1 validate_restored_directory "$runtime_parent" runtime_parent || rollback_rc=1 validate_restored_directory "$runtime_bin" runtime_bin || rollback_rc=1 + validate_restored_directory "$runtime_bin/gcloud-config" gcloud_config || rollback_rc=1 validate_restored_directory "$dropin_dir" dropin_dir || rollback_rc=1 fi if [ "$rollback_rc" -eq 0 ]; then @@ -705,7 +1247,7 @@ rollback_runtime() { fi if [ "$rollback_rc" -eq 0 ]; then sudo systemctl start "$SERVICE" || rollback_rc=1 - assert_service_running || rollback_rc=1 + assert_service_running false || rollback_rc=1 fi return "$rollback_rc" } @@ -728,6 +1270,7 @@ on_exit() { "$tool_stage" \ "$permission_verifier_stage" \ "$service_env_verifier_stage" \ + "$server_ca_stage" \ "$revision_stage" \ "$dropin_stage" || true sudo rm -rf -- "$remote_tmp" || true @@ -754,11 +1297,13 @@ fi sudo chown -R root:root "$payload_dir" sudo find "$payload_dir" -type d -exec chmod 0555 {} + sudo find "$payload_dir" -type f -exec chmod 0444 {} + +sudo install -d -o root -g root -m 0555 "$payload_dir/gcloud-config" sudo chmod 0555 \ "$payload_dir/$WRAPPER_REL" \ "$payload_dir/$TOOL_REL" \ "$payload_dir/$PERMISSION_VERIFIER_REL" \ "$payload_dir/$SERVICE_ENV_VERIFIER_REL" +test "$(sudo sha256sum "$payload_dir/$SERVER_CA_REL" | awk '{print $1}')" = "$SERVER_CA_SHA" test "$(sudo sha256sum "$payload_dir/$WRAPPER_REL" | awk '{print $1}')" = "$WRAPPER_SHA" test "$(sudo sha256sum "$payload_dir/$TOOL_REL" | awk '{print $1}')" = "$TOOL_SHA" test "$(sudo sha256sum "$payload_dir/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" @@ -783,6 +1328,7 @@ for existing_directory in \ "$runtime_grandparent" \ "$runtime_parent" \ "$runtime_bin" \ + "$runtime_bin/gcloud-config" \ "$dropin_dir"; do assert_existing_root_directory "$existing_directory" done @@ -791,19 +1337,24 @@ for existing_file in \ "$runtime_bin/cloudsql_memory_tool.py" \ "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" \ "$runtime_bin/verify_gcp_leoclean_service_environment.py" \ + "$runtime_bin/cloudsql-server-ca.pem" \ "$runtime_bin/.livingip-runtime-revision" \ "$dropin_dir/$DROPIN_NAME"; do assert_existing_root_file "$existing_file" done +if sudo test -d "$runtime_bin/gcloud-config"; then + sudo test -z "$(sudo find "$runtime_bin/gcloud-config" -mindepth 1 -print -quit)" +fi -assert_service_running -preflight_service_before="$(service_fingerprint)" -assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false -run_scoped_status pre "$payload_dir/$TOOL_REL" -assert_administrator_secret_denied -run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" -assert_service_fingerprint "$preflight_service_before" +assert_service_running false if [ "$PREFLIGHT_ONLY" = true ]; then + preflight_service_before="$(service_fingerprint)" + assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false + run_scoped_status pre "$payload_dir/$TOOL_REL" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" + assert_administrator_secret_denied + run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" + assert_service_fingerprint "$preflight_service_before" + assert_service_running false systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager exit 0 fi @@ -812,17 +1363,29 @@ backup_path "$runtime_bin/teleo-kb" wrapper backup_path "$runtime_bin/cloudsql_memory_tool.py" tool backup_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier backup_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier +backup_path "$runtime_bin/cloudsql-server-ca.pem" server_ca backup_path "$dropin_dir/$DROPIN_NAME" dropin backup_path "$runtime_bin/.livingip-runtime-revision" revision backup_directory_state "$runtime_parent" runtime_parent backup_directory_state "$runtime_grandparent" runtime_grandparent backup_directory_state "$runtime_bin" runtime_bin +backup_directory_state "$runtime_bin/gcloud-config" gcloud_config backup_directory_state "$dropin_dir" dropin_dir +preflight_service_before="$(service_fingerprint)" +assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false +run_scoped_status pre "$payload_dir/$TOOL_REL" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" +assert_administrator_secret_denied +run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" +assert_service_fingerprint "$preflight_service_before" +assert_service_running false mutation_started=true sudo systemctl stop "$SERVICE" sudo install -d -o root -g root -m 0755 "$runtime_parent" sudo install -d -o root -g root -m 0755 "$runtime_bin" +sudo install -d -o root -g root -m 0555 "$runtime_bin/gcloud-config" +sudo install -o root -g root -m 0644 "$payload_dir/$SERVER_CA_REL" "$server_ca_stage" +sudo mv -f -- "$server_ca_stage" "$runtime_bin/cloudsql-server-ca.pem" sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage" sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage" sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" "$permission_verifier_stage" @@ -845,10 +1408,13 @@ sudo systemctl start "$SERVICE" assert_deployed_files assert_service_running +post_service_before="$(service_fingerprint)" assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py" run_service_wrapper_status post assert_administrator_secret_denied run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" "$work_dir/permission-post/receipt.json" +assert_service_fingerprint "$post_service_before" +assert_service_running systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager mutation_started=false REMOTE @@ -860,4 +1426,5 @@ COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command" diff --git a/docs/gcp-leoclean-runtime-reconciliation.md b/docs/gcp-leoclean-runtime-reconciliation.md index eafcb50..3ac69de 100644 --- a/docs/gcp-leoclean-runtime-reconciliation.md +++ b/docs/gcp-leoclean-runtime-reconciliation.md @@ -44,6 +44,11 @@ IAM or networking. 7. Deployment proves the attached VM service account from the metadata server, uses an empty phase-local Cloud SDK configuration, and rolls the complete prior runtime set back if installation or post-restart verification fails. +8. Post-restart status, permission, and administrator-secret denial probes join + both the actual service mount and network namespaces. Effective systemd + network, address-family, socket-bind, syscall-filter, and LSM properties must + match the reviewed unrestricted transport contract so a host-context helper + cannot produce a false runtime receipt. ## Safe Order Of Operations @@ -70,21 +75,33 @@ an artifact, or a command transcript. ### 4. Provision the scoped PostgreSQL role once -Run `ops/gcp_leoclean_runtime_role.sql` as the Cloud SQL administrator from the -private VM path. Supply the new runtime password through -`TELEO_LEOCLEAN_DB_PASSWORD`; the SQL file does not contain it. The migration: +Run `ops/provision_gcp_leoclean_runtime_role.sh` as the Cloud SQL administrator +from the private VM path. Supply the new runtime password through +`TELEO_LEOCLEAN_DB_PASSWORD` and the administrator password through +`PGPASSWORD`. The wrapper captures and unsets both fields before its first +external child, pins the private endpoint and reviewed server CA, detaches +`psql` from the controlling terminal, and feeds the runtime value only to +psql's client-side `\password` prompt. The SQL file and argv never contain the +cleartext runtime password. The migration: - creates or rotates `leoclean_kb_runtime`; -- keeps `leoclean_kb_runtime` `NOLOGIN` while both the `teleo_canonical` grant - phase and the legacy `teleo_kb` denial-verification phase run, and enables - `LOGIN` only in the final transaction after both phases pass; +- changes any pre-existing `leoclean_kb_runtime` to `NOLOGIN`, removes its + membership edges, and terminates its existing sessions before password + handling; an interruption therefore leaves the old login disabled; +- enables `LOGIN` only as the last statement in the same transaction as the + database ACL, canonical grant, routine, large-object, and topology checks; - creates a dedicated `NOLOGIN` function owner with no role memberships; - removes stale table, column, sequence, function, and role-membership grants; +- inventories existing principals, replaces `PUBLIC CONNECT` with explicit + preserved grants, gives the runtime `CONNECT` only to `teleo_canonical`, and + leaves the staging owner with no database connection target; +- inventories and preserves existing non-scoped application-routine and + large-object access while removing `PUBLIC` and both scoped roles from all + other application routines and every persistent large-object mutator; - grants exact canonical reads; - creates a locked security-definer staging function that hard-codes both `pending_review` and canonical proposer `leo`; - grants no direct table writes; -- removes all access to the legacy `teleo_restore` schema; - aborts if either scoped role owns or can reach anything outside the explicit allowlist. @@ -99,11 +116,13 @@ by a `pg_catalog, pg_temp` search path, schema-qualified relations, a fixed The administrator password is used only for this bounded bootstrap. Retain no password output. -`teleo_kb` can still be connectable through its existing `PUBLIC CONNECT` -grant. That database-wide default is not treated as runtime authority. The -migration denies the scoped role access to the `teleo_restore` schema, the -runtime client is pinned to `teleo_canonical`, and audit fallback remains -disabled with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`. +`teleo_kb`, `template1`, and every other noncanonical database are actual +negative connection checks. PostgreSQL's startup protocol does not expose a +SQLSTATE through `psql`, so the verifier requires the exact C-locale +`permission denied for database` and `does not have CONNECT privilege` +diagnostics in addition to the catalog proof that the runtime's sole effective +target is `teleo_canonical`. Audit fallback remains disabled with +`TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`. ### 5. Preflight and deploy the merged runtime @@ -143,41 +162,71 @@ directory as a read-only bind at its existing `/home/teleo/.hermes/profiles/leoclean/bin` path; verification compares the source and service-namespace inode and hashes and proves the mount is read-only. To prevent the `teleo` service user from renaming a writable ancestor and -recreating that absolute path, the service namespace mounts -`/home/teleo/.hermes` read-only, allows writes back only to the profile's -`state` and `workspace` subdirectories, and removes `CAP_SYS_ADMIN`. Live -verification requires `.hermes` and `bin` to be read-only, both allowed -subdirectories to be writable, `/home` to be non-writable by `teleo`, and the -effective capability set to exclude `CAP_SYS_ADMIN`. +recreating that absolute path, the service namespace mounts the complete +`/home/teleo` tree read-only, allows writes back only to the profile's `state` +and `workspace` subdirectories, clears every bounding and ambient capability, +and enforces `NoNewPrivileges`. Live verification requires the service home and +bound `bin` to be read-only, both allowed subdirectories to be writable, the +exact `teleo` group set, and every process capability field to be zero. Preflight, post-restart, and `--verify-only` checks inspect the environment of the actual systemd `MainPID` through `/proc//environ`, rather than trusting the -configured unit environment alone. A standalone fail-closed verifier requires +configured unit environment alone. It also rejects effective credential +properties, auxiliary `Exec*` hooks, any `EnvironmentFile`, and any drop-in +ordered after the reviewed configuration. A standalone fail-closed verifier requires the exact reviewed private target and runtime role, requires `TELEO_GCP_METADATA_ONLY=1`, and rejects all PostgreSQL, Cloud SDK, proxy, TLS trust, broader Google credential, shell-startup, dynamic-loader, Python-loader, -and administrator-secret overrides without printing their values. Effective -`EnvironmentFile` use and drop-ins ordered after the reviewed configuration are -rejected before restart. This blocks +administrator-secret, excerpt, credential-mode, claim-base, and retry-count +overrides without printing their values. The reviewed root-owned empty +`CLOUDSDK_CONFIG`, pinned `PGSSLMODE=verify-ca`, +server CA, and Python isolation fields must match exactly. This blocks pre-wrapper injection through fields such as `BASH_ENV`, any `LD_*`, `PYTHONPATH`, `PYTHONHTTPSVERIFY`, or `SSLKEYLOGFILE`. The checks also require the reviewed drop-in to appear in `DropInPaths`. The wrapper status probe then imports the validated environment from the actual `MainPID` with `nsenter ---env`, enters its mount namespace, drops to `teleo`, and invokes the bound +--env`, enters its mount and network namespaces, drops to `teleo`, and invokes the bound wrapper without reinjecting any database identity setting. +The effective unit must use the host network namespace (`PrivateNetwork=no`), +have no `NetworkNamespacePath`, `IPAddressDeny`, `IPAddressAllow`, +`RestrictAddressFamilies`, `SocketBindAllow`, `SocketBindDeny`, +`RestrictNetworkInterfaces`, `SystemCallFilter`, `AppArmorProfile`, or +`SELinuxContext` override, and load no +later drop-in. The executable regression harness injects each conflicting +property and requires the gate to fail. These checks cover restrictions that a +new `nsenter` process would not inherit merely by sharing the service network +namespace. + +For cgroup/eBPF restrictions, post-restart probes do not rely on enumeration. +A root-only bounded runner verifies the service's single cgroup-v2 membership +against systemd's effective `ControlGroup`, forks a stopped child, moves only +that child into the service cgroup, confirms the move, and then resumes it to +join the service namespaces and drop to `teleo`. It propagates exit or signal +status and reaps the child on every path. The child installs a Linux +parent-death `SIGKILL` before stopping, with a parent-PID race check; the parent +blocks and handles `INT`, `TERM`, and `HUP` through cleanup before re-raising +the signal. Before resume, every supported Linux `RLIMIT_*` soft/hard pair is +copied from the live `MainPID` with a stable source and child readback. Live +`MainPID` verification also requires seccomp mode and filter count zero, the +exact `unconfined` LSM context, `PrivateUsers=no`, and the same user-namespace +inode as PID 1, preventing a per-process filter, profile, resource ceiling, or +user namespace from being mistaken for the reviewed service context. + The service runtime bypasses Cloud SDK configuration and credential caches entirely. The helper obtains the attached service-account email and a bounded access token from the fixed GCE metadata endpoint, requires the exact expected service account, and accesses only the named scoped Secret Manager version over the fixed HTTPS API. Its URL opener disables proxies. The scoped status read and administrator-secret IAM denial are rerun after restart. The denial probe -imports the actual systemd `MainPID` environment and mount namespace, drops to +imports the actual systemd `MainPID` environment and mount and network +namespaces, drops to the service's `teleo` identity, and must observe the exact `secretmanager.versions.access` permission denial. No token or -secret value is retained. The status read executes the bound `teleo-kb` wrapper -from inside the service mount namespace, proving the wrapper and its adjacent -helper rather than bypassing them with a direct helper invocation. A failure +secret value is retained. The status read and permission verifier execute from +inside the service mount and network namespaces; the status path invokes the +bound `teleo-kb` wrapper and its adjacent helper rather than bypassing them with +a direct helper invocation. A failure after mutation restores the prior runtime/drop-in directory existence and metadata, wrapper, helper, both verifiers, drop-in, and revision as one set, validates it against the root-only backup, and only then reloads systemd and @@ -196,7 +245,7 @@ sudo chmod 0700 "$receipt_dir" run_id="$(date -u +%Y%m%d%H%M%S)-manual" sudo -n -u teleo env -i HOME=/home/teleo \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - python3 /usr/local/libexec/livingip/leoclean-kb/verify_gcp_leoclean_runtime_permissions.py \ + /usr/bin/python3 -I /usr/local/libexec/livingip/leoclean-kb/verify_gcp_leoclean_runtime_permissions.py \ --run-id "$run_id" \ --output "$receipt_dir/permission-receipt.json" sudo stat -c '%U:%G:%a' "$receipt_dir/permission-receipt.json" @@ -230,13 +279,17 @@ Retain a redacted receipt containing: both before and after the transaction; - denied direct insert, update, and delete on `kb_stage.kb_proposals`; - denied insert, update, and delete on canonical `public.claims`; +- denied `lo_creat`, `lo_create`, and `lo_from_bytea`, zero effective + large-object mutator execution, and zero scoped large-object ownership/ACLs; - denied reviewer/apply security-definer functions; - exact function-catalog posture: only the five-argument staging function is executable, the forged six-argument overload is absent, and each expected reviewer/apply function exists but is not executable by the runtime role; - unavailable forged-proposer overload and denied `SET ROLE` escalation into the staging owner, reviewer, apply, or administrator roles; -- denied access to `teleo_restore` even when `teleo_kb` remains connectable; +- denied startup connections to both `teleo_kb` and `template1`, with the + catalog showing `teleo_canonical` as the sole runtime connection target and + zero remaining `PUBLIC CONNECT` grants; - readable scoped runtime secret and an IAM permission denial for the administrator secret, with neither secret value retained; - zero Telegram messages and zero committed canary rows. diff --git a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py index 3367207..9c579fd 100755 --- a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py +++ b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 -I """Cloud SQL memory and KB-proposal bridge for the GCP leoclean profile. Canonical claim/strategy application remains review-gated. The production-like @@ -17,8 +17,10 @@ import hashlib import json import os import re +import stat import subprocess import urllib.request +from pathlib import Path from typing import Any STOPWORDS = { @@ -84,6 +86,12 @@ RUNTIME_GCP_PROJECT = "teleo-501523" RUNTIME_CLOUDSQL_HOST = "10.61.0.3" RUNTIME_CLOUDSQL_PORT = "5432" RUNTIME_CLOUDSQL_DB = "teleo_canonical" +RUNTIME_SYSTEM_IDENTIFIER = "7659718422914359312" +RUNTIME_CLOUDSDK_CONFIG = "/usr/local/libexec/livingip/leoclean-kb/gcloud-config" +RUNTIME_PSQL_BIN = "/usr/bin/psql" +RUNTIME_SSL_MODE = "verify-ca" +RUNTIME_SSL_ROOT_CERT = "/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem" +RUNTIME_SSL_ROOT_CERT_SHA256 = "80701e768f0e1f6b9d621aa0b53f6e851daaa276c6d9a8e51a300fbc015539cb" RUNTIME_SERVICE_ACCOUNT = "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" GCE_METADATA_EMAIL_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" GCE_METADATA_TOKEN_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" @@ -105,13 +113,60 @@ RUNTIME_TLS_TRUST_ENVIRONMENT = frozenset( "SSL_CERT_DIR", "SSL_CERT_FILE", "SSLKEYLOGFILE", + "OPENSSL_CONF", + "OPENSSL_ENGINES", + "OPENSSL_MODULES", + "GCONV_PATH", } ) RUNTIME_PSQL_ENV_BASE = { "PATH": "/usr/bin:/bin", - "PGSSLMODE": "require", + "PGSSLMODE": RUNTIME_SSL_MODE, "PGCONNECT_TIMEOUT": "8", } + + +def runtime_ssl_root_cert() -> str: + """Return the reviewed installed or ephemeral preflight CA path.""" + + raw = os.environ.get("TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT", RUNTIME_SSL_ROOT_CERT) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + if resolved != candidate or not resolved.is_file(): + raise OSError + for part in (resolved, *resolved.parents): + metadata = part.stat() + if metadata.st_uid != 0 or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise OSError + certificate = resolved.read_bytes() + except OSError: + raise SystemExit("Reviewed Cloud SQL server CA path is unavailable or untrusted.") from None + if hashlib.sha256(certificate).hexdigest() != RUNTIME_SSL_ROOT_CERT_SHA256: + raise SystemExit("Reviewed Cloud SQL server CA does not match the pinned certificate.") + return str(resolved) + + +def runtime_cloudsdk_config() -> str: + """Return a root-owned empty SDK configuration directory.""" + + raw = os.environ.get("TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG", RUNTIME_CLOUDSDK_CONFIG) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + if resolved != candidate or not resolved.is_dir() or any(resolved.iterdir()): + raise OSError + for part in (resolved, *resolved.parents): + metadata = part.stat() + if metadata.st_uid != 0 or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise OSError + except OSError: + raise SystemExit("Reviewed empty Cloud SDK configuration is unavailable or untrusted.") from None + return str(resolved) CLONE_READ_ONLY_PGOPTIONS = "-c default_transaction_read_only=on" RECEIPTED_READ_COMMANDS = frozenset( { @@ -290,6 +345,8 @@ def validate_runtime_connection(args: argparse.Namespace) -> None: password_secret = str(args.password_secret or "") mode = str(getattr(args, "credential_mode", "runtime")) if mode == "runtime": + ssl_root_cert = runtime_ssl_root_cert() + cloudsdk_config = runtime_cloudsdk_config() if user != RUNTIME_DB_USER: raise SystemExit(f"Leo runtime requires database user {RUNTIME_DB_USER}.") if password_secret != RUNTIME_PASSWORD_SECRET: @@ -310,12 +367,31 @@ def validate_runtime_connection(args: argparse.Namespace) -> None: if os.environ.get("TELEO_GCP_METADATA_ONLY") != "1": raise SystemExit("Leo runtime requires TELEO_GCP_METADATA_ONLY=1.") forbidden_environment = any( - normalized_key.startswith("PG") - or normalized_key.startswith("CLOUDSDK_") + ( + normalized_key.startswith("PG") + and not ( + (normalized_key == "PGSSLMODE" and value == RUNTIME_SSL_MODE) + or (normalized_key == "PGSSLROOTCERT" and value == ssl_root_cert) + ) + ) + or ( + normalized_key.startswith("CLOUDSDK_") + and not ( + normalized_key == "CLOUDSDK_CONFIG" + and value == cloudsdk_config + ) + ) or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS" or normalized_key.startswith("LD_") or normalized_key.startswith("BASH_FUNC_") - or (normalized_key.startswith("PYTHON") and normalized_key != "PYTHONUNBUFFERED") + or ( + normalized_key.startswith("PYTHON") + and not ( + (normalized_key == "PYTHONNOUSERSITE" and value == "1") + or (normalized_key == "PYTHONSAFEPATH" and value == "1") + or normalized_key == "PYTHONUNBUFFERED" + ) + ) or normalized_key in { "BASH_ENV", "BASHOPTS", @@ -329,7 +405,8 @@ def validate_runtime_connection(args: argparse.Namespace) -> None: or normalized_key == "PROXY" or normalized_key.endswith("_PROXY") or normalized_key in RUNTIME_TLS_TRUST_ENVIRONMENT - for normalized_key in (key.upper() for key in os.environ) + for key, value in os.environ.items() + for normalized_key in (key.upper(),) ) if forbidden_environment: raise SystemExit( @@ -515,6 +592,7 @@ def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: credential = password(args) if args.credential_mode == "runtime": env = dict(RUNTIME_PSQL_ENV_BASE) + env["PGSSLROOTCERT"] = runtime_ssl_root_cert() else: env = {key: value for key, value in os.environ.items() if not key.startswith("PG")} env.update( @@ -529,7 +607,7 @@ def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: if args.credential_mode == "clone-readonly": env["PGOPTIONS"] = CLONE_READ_ONLY_PGOPTIONS result = subprocess.run( - ["psql", "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"], + [RUNTIME_PSQL_BIN, "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"], text=True, capture_output=True, input=sql, @@ -583,6 +661,8 @@ select jsonb_build_object( raise SystemExit("Canonical database read marker returned an unexpected server port.") if marker.get("ssl") is not True: raise SystemExit("Canonical database read marker did not prove an encrypted connection.") + if marker.get("system_identifier") != RUNTIME_SYSTEM_IDENTIFIER: + raise SystemExit("Canonical database read marker returned an unexpected system identifier.") return marker diff --git a/hermes-agent/leoclean-bin/teleo-kb-gcp b/hermes-agent/leoclean-bin/teleo-kb-gcp new file mode 100755 index 0000000..9e56c18 --- /dev/null +++ b/hermes-agent/leoclean-bin/teleo-kb-gcp @@ -0,0 +1,52 @@ +#!/bin/bash +set -euo pipefail + +# This wrapper is installed only into the GCP service's root-controlled bind. +# Ignore per-invocation mode and endpoint overrides so a service child cannot +# select the VPS/local backend or a different database credential context. +SCRIPT_PATH=${BASH_SOURCE[0]} +SCRIPT_DIR=${SCRIPT_PATH%/*} +if [ "$SCRIPT_DIR" = "$SCRIPT_PATH" ]; then + SCRIPT_DIR=. +fi +SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)" +CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py" + +for argument in "$@"; do + case "$argument" in + --host|--host=*|--port|--port=*|--db|--db=*|--canonical-db|--canonical-db=*|\ + --user|--user=*|--password-secret|--password-secret=*|--project|--project=*|\ + --credential-mode|--credential-mode=*) + echo "teleo-kb: GCP runtime identity and endpoint overrides are forbidden" >&2 + exit 64 + ;; + esac +done + +if [ ! -x "$CLOUDSQL_TOOL" ]; then + echo "teleo-kb: reviewed GCP Cloud SQL tool is unavailable" >&2 + exit 69 +fi +if [ ! -x /usr/bin/python3 ] || [ ! -x /usr/bin/psql ]; then + echo "teleo-kb: required GCP runtime dependency is unavailable" >&2 + exit 69 +fi + +exec /usr/bin/env -i \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + LANG=C LC_ALL=C \ + PYTHONNOUSERSITE=1 PYTHONSAFEPATH=1 \ + CLOUDSDK_CONFIG=/usr/local/libexec/livingip/leoclean-kb/gcloud-config \ + PGSSLMODE=verify-ca \ + PGSSLROOTCERT=/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem \ + TELEO_KB_MODE=cloudsql \ + TELEO_GCP_METADATA_ONLY=1 \ + TELEO_GCP_PROJECT=teleo-501523 \ + TELEO_CLOUDSQL_HOST=10.61.0.3 \ + TELEO_CLOUDSQL_PORT=5432 \ + TELEO_CLOUDSQL_DB=teleo_canonical \ + TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical \ + TELEO_CLOUDSQL_USER=leoclean_kb_runtime \ + TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \ + TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \ + /usr/bin/python3 -I "$CLOUDSQL_TOOL" "$@" diff --git a/ops/gcp-teleo-pgvector-standby-server-ca.pem b/ops/gcp-teleo-pgvector-standby-server-ca.pem new file mode 100644 index 0000000..e31490d --- /dev/null +++ b/ops/gcp-teleo-pgvector-standby-server-ca.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDcTCCAlmgAwIBAgIBADANBgkqhkiG9w0BAQsFADBwMS0wKwYDVQQuEyQzNTcw +NmEzOC03MGRjLTQ3YTQtOWExOS04NGMxYTk0YzQ3ZjMxHDAaBgNVBAMTE0Nsb3Vk +IFNRTCBTZXJ2ZXIgQ0ExFDASBgNVBAoTC0dvb2dsZSwgSW5jMQswCQYDVQQGEwJV +UzAeFw0yNjA3MDcwOTM3NTJaFw0zNjA3MDQwOTM4NTJaMHAxLTArBgNVBC4TJDM1 +NzA2YTM4LTcwZGMtNDdhNC05YTE5LTg0YzFhOTRjNDdmMzEcMBoGA1UEAxMTQ2xv +dWQgU1FMIFNlcnZlciBDQTEUMBIGA1UEChMLR29vZ2xlLCBJbmMxCzAJBgNVBAYT +AlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0HwhrUzFIi9bqPbC +9FQjCalxEVYsu7tw+YzBtUaKOy+u/Y1e6TKJzFng/9wS8pXgY1Ul1/mn1S205yKR +5ydEESq7UL4pp7onSb2vGeu4NDTXDbcimZz++35dPbhFw7wtm1QSRMBHsONEDGaS +7/qIB3EN79SS9GxbSUZNsZOAn4DM9RSahtQrXa6NLlP7I9gPxvqEKRjFqUcBkWsE +Qzoz/yyxYQybZQo1UR9XCbGWFphi85cW15ITpfs7czoOVUk39bPNJKAVQFRqPUWK +xBC+1pHb7fe7xYk5j3jJbtoL6ixunQIFDCGRCy7/SEfIj5dWriDajVHwctilUmu1 +Xy/A0QIDAQABoxYwFDASBgNVHRMBAf8ECDAGAQH/AgEAMA0GCSqGSIb3DQEBCwUA +A4IBAQBi8MoiT4kJAut4mLPibKldRB9s2Of7Jw8PmKNS50BIOZf8zrSrNMX5yC2i +/y9nmc8k9OCIYbzC6F/mTACmtHM1phoBtPYDjWYp+YS9cdo0HHGU2hwEP7PTfEOg +11ua2J1gjFbgVZVb7qNGV4n1QjCcAh1hWnJ2gLa55hiZBGZ0fuHTwBagWwRtom+u +YdD1d1NETInnCxnhzu7q+r0sjFUJABIj4qU/q/uVMHaAuy4Zy4PMctQofKSJ4ixf +vbU2/Sm7cbzgzcg2TJwpSqdDM/EdDMi5JtmMyZnRoKLZjH3CYI1091dyTBdk8RhK +MB4pqjJurXS8xKeA9dQjuvnIQvot +-----END CERTIFICATE----- diff --git a/ops/gcp_leoclean_runtime_role.sql b/ops/gcp_leoclean_runtime_role.sql index da884c8..5962180 100644 --- a/ops/gcp_leoclean_runtime_role.sql +++ b/ops/gcp_leoclean_runtime_role.sql @@ -1,7 +1,8 @@ \set ON_ERROR_STOP on --- One-time GCP staging provisioning for Leo's Cloud SQL identity. The password --- is supplied only through the process environment and is never stored here. +-- One-time GCP staging provisioning for Leo's Cloud SQL identity. Run this +-- only through ops/provision_gcp_leoclean_runtime_role.sh; psql's \password +-- command encrypts the supplied value client-side, so cleartext is never SQL. -- -- The login can read the canonical allowlist and can stage a pending proposal -- through one narrowly-scoped SECURITY DEFINER function. It cannot inherit or @@ -24,19 +25,34 @@ alter role leoclean_kb_stage_owner with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1 password null; --- Rotate the credential only after an idempotent rerun has disabled any --- pre-existing LOGIN role. This ordering has an intentional autocommit --- boundary: interruption can leave NOLOGIN, never a still-enabled old role. -\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD -select format('alter role leoclean_kb_runtime password %L', :'runtime_password') -\gexec +-- Snapshot every non-superuser principal connected to either scoped role before +-- removing the direct membership edges. Existing sessions can retain an +-- already-selected SET ROLE identity after NOLOGIN/revocation, so all other +-- connected sessions must be terminated before password or privilege work. +create temporary table leoclean_fenced_roles ( + role_oid oid primary key, + role_name name unique not null +) on commit preserve rows; -alter role leoclean_kb_runtime reset all; -alter role leoclean_kb_stage_owner reset all; +with recursive connected(role_oid) as ( + values + ('leoclean_kb_runtime'::pg_catalog.regrole::oid), + ('leoclean_kb_stage_owner'::pg_catalog.regrole::oid) + union + select membership.member + from connected + join pg_catalog.pg_auth_members membership + on membership.roleid = connected.role_oid +) +insert into pg_temp.leoclean_fenced_roles (role_oid, role_name) +select role_row.oid, role_row.rolname + from connected + join pg_catalog.pg_roles role_row on role_row.oid = connected.role_oid + where not role_row.rolsuper; --- NOINHERIT does not prevent SET ROLE. Remove every membership edge in either --- direction so neither scoped principal can reach, or be reached through, a --- broader role left behind by an earlier deployment. +-- NOINHERIT does not prevent SET ROLE. Remove every direct membership edge in +-- either direction before terminating the snapshotted sessions, closing the +-- race for newly connected membership principals. select format('revoke %I from %I', granted_role.rolname, member_role.rolname) from pg_catalog.pg_auth_members membership join pg_catalog.pg_roles granted_role on granted_role.oid = membership.roleid @@ -45,11 +61,62 @@ select format('revoke %I from %I', granted_role.rolname, member_role.rolname) or member_role.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') \gexec -revoke all privileges on database teleo_canonical from leoclean_kb_runtime; -revoke all privileges on database teleo_canonical from leoclean_kb_stage_owner; -revoke all privileges on database teleo_kb from leoclean_kb_runtime; -revoke all privileges on database teleo_kb from leoclean_kb_stage_owner; -grant connect on database teleo_canonical to leoclean_kb_runtime; +do $session_fence$ +declare + target record; + remaining text; +begin + for target in + select activity.pid + from pg_catalog.pg_stat_activity activity + join pg_temp.leoclean_fenced_roles fenced on fenced.role_name = activity.usename + where activity.pid <> pg_catalog.pg_backend_pid() + and activity.backend_type = 'client backend' + loop + if not pg_catalog.pg_terminate_backend(target.pid, 5000) then + raise exception 'could not terminate a pre-existing scoped-role session'; + end if; + end loop; + + select pg_catalog.string_agg(activity.pid::text, ',' order by activity.pid) + into remaining + from pg_catalog.pg_stat_activity activity + join pg_temp.leoclean_fenced_roles fenced on fenced.role_name = activity.usename + where activity.pid <> pg_catalog.pg_backend_pid() + and activity.backend_type = 'client backend'; + if remaining is not null then + raise exception 'pre-existing scoped-role sessions remain after termination'; + end if; + + select pg_catalog.string_agg(prepared.gid, ',' order by prepared.gid) + into remaining + from pg_catalog.pg_prepared_xacts prepared + join pg_temp.leoclean_fenced_roles fenced on fenced.role_name = prepared.owner; + if remaining is not null then + raise exception 'scoped-role prepared transactions must be resolved before provisioning'; + end if; +end +$session_fence$; + +-- This is deliberately after the autocommitted NOLOGIN/session/membership +-- fence. Interruption can leave a disabled role, never a still-enabled old +-- login or an old connected session. The wrapper supplies the two prompts. +\password leoclean_kb_runtime + +alter role leoclean_kb_runtime reset all; +alter role leoclean_kb_stage_owner reset all; +drop table pg_temp.leoclean_fenced_roles; + +select pg_catalog.format( + 'alter role %I in database %I reset all', + role_row.rolname, + database_row.datname + ) + from pg_catalog.pg_db_role_setting setting_row + join pg_catalog.pg_roles role_row on role_row.oid = setting_row.setrole + join pg_catalog.pg_database database_row on database_row.oid = setting_row.setdatabase + where role_row.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') +\gexec -- PostgreSQL normally grants TEMP to PUBLIC. A role-specific REVOKE cannot -- override that inherited grant, while REVOKE TEMP FROM PUBLIC would be a @@ -77,6 +144,47 @@ alter role leoclean_kb_runtime in database teleo_canonical begin; +-- CONNECT granted to PUBLIC cannot be denied to one role with a role-specific +-- REVOKE. Preserve every existing non-scoped principal's effective CONNECT +-- set as explicit ACLs, remove the cluster-wide PUBLIC grant from every +-- database (including templates), and then grant the runtime its sole target. +-- This runs in the same transaction as every remaining privilege/topology +-- assertion, so a later failure restores the original ACLs atomically. +create temporary table leoclean_preserved_database_connect ( + role_name name not null, + database_name name not null, + primary key (role_name, database_name) +) on commit drop; + +insert into pg_temp.leoclean_preserved_database_connect (role_name, database_name) +select role_row.rolname, database_row.datname + from pg_catalog.pg_roles role_row + cross join pg_catalog.pg_database database_row + where role_row.rolname not in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and pg_catalog.has_database_privilege(role_row.oid, database_row.oid, 'CONNECT'); + +select pg_catalog.format('revoke connect on database %I from public', database_row.datname) + from pg_catalog.pg_database database_row +\gexec + +select pg_catalog.format( + 'revoke all privileges on database %I from leoclean_kb_runtime, leoclean_kb_stage_owner', + database_row.datname + ) + from pg_catalog.pg_database database_row +\gexec + +select pg_catalog.format( + 'grant connect on database %I to %I', + preserved.database_name, + preserved.role_name + ) + from pg_temp.leoclean_preserved_database_connect preserved + order by preserved.database_name, preserved.role_name +\gexec + +grant connect on database teleo_canonical to leoclean_kb_runtime; + -- The deploy principal needs temporary ownership authority to replace an -- existing function on idempotent reruns. This membership is transaction-local -- and is revoked before verification/commit. @@ -95,6 +203,114 @@ revoke all privileges on all functions in schema public, kb_stage revoke all privileges on all procedures in schema public, kb_stage from leoclean_kb_runtime, leoclean_kb_stage_owner; +-- Built-in defaults also grant PUBLIC EXECUTE on newly created application +-- routines. Preserve every existing non-scoped role's effective access, +-- remove PUBLIC/scoped execution from the complete application routine set, +-- and restore the inventory explicitly. The reviewed staging function is +-- normalized again after CREATE OR REPLACE below. +create temporary table leoclean_preserved_app_routine_execute ( + role_name name not null, + function_oid oid not null, + primary key (role_name, function_oid) +) on commit drop; + +insert into pg_temp.leoclean_preserved_app_routine_execute (role_name, function_oid) +select role_row.rolname, function_row.oid + from pg_catalog.pg_roles role_row + cross join pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where role_row.rolname not in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and namespace.nspname in ('public', 'kb_stage') + and pg_catalog.has_function_privilege(role_row.oid, function_row.oid, 'EXECUTE'); + +select pg_catalog.format( + 'revoke execute on %s %I.%I(%s) from public, leoclean_kb_runtime, leoclean_kb_stage_owner', + case when function_row.prokind = 'p' then 'procedure' else 'function' end, + namespace.nspname, + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid) + ) + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname in ('public', 'kb_stage') +\gexec + +select pg_catalog.format( + 'grant execute on %s %I.%I(%s) to %I', + case when function_row.prokind = 'p' then 'procedure' else 'function' end, + namespace.nspname, + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid), + preserved.role_name + ) + from pg_temp.leoclean_preserved_app_routine_execute preserved + join pg_catalog.pg_proc function_row on function_row.oid = preserved.function_oid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + order by namespace.nspname, function_row.proname, preserved.role_name +\gexec + +select pg_catalog.format( + 'revoke all privileges on large object %s from leoclean_kb_runtime, leoclean_kb_stage_owner', + metadata.oid + ) + from pg_catalog.pg_largeobject_metadata metadata +\gexec + +-- Default PostgreSQL grants EXECUTE on built-in large-object mutators to +-- PUBLIC, which would let the runtime persist data despite all relation ACLs. +-- Snapshot and explicitly preserve every existing non-scoped role's effective +-- access, then remove PUBLIC and both scoped roles from every mutator overload. +create temporary table leoclean_preserved_lo_execute ( + role_name name not null, + function_oid oid not null, + primary key (role_name, function_oid) +) on commit drop; + +insert into pg_temp.leoclean_preserved_lo_execute (role_name, function_oid) +select role_row.rolname, function_row.oid + from pg_catalog.pg_roles role_row + cross join pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where role_row.rolname not in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and pg_catalog.has_function_privilege(role_row.oid, function_row.oid, 'EXECUTE'); + +select pg_catalog.format( + 'revoke execute on function pg_catalog.%I(%s) from public, leoclean_kb_runtime, leoclean_kb_stage_owner', + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid) + ) + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) +\gexec + +select pg_catalog.format( + 'grant execute on function pg_catalog.%I(%s) to %I', + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid), + preserved.role_name + ) + from pg_temp.leoclean_preserved_lo_execute preserved + join pg_catalog.pg_proc function_row on function_row.oid = preserved.function_oid + order by function_row.proname, preserved.role_name +\gexec + +select pg_catalog.format( + 'revoke all privileges on parameter %I from leoclean_kb_runtime, leoclean_kb_stage_owner', + parameter_acl.parname + ) + from pg_catalog.pg_parameter_acl parameter_acl +\gexec + -- Table-level REVOKE does not clear column ACLs. Clear every column ACL for both -- principals before rebuilding the exact allowlist below. select format( @@ -310,6 +526,264 @@ begin raise exception 'scoped Leo roles retain role memberships: %', unexpected; end if; + if not exists ( + select 1 + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' + and relation.relkind = 'r' + and relation.relpersistence = 'p' + and not relation.relrowsecurity + and not relation.relforcerowsecurity + ) then + raise exception 'kb_stage.kb_proposals is not the reviewed permanent base table'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, relation.relkind), + ', ' + order by namespace.nspname, relation.relname + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where (namespace.nspname, relation.relname) in ( + ('public', 'agents'), ('public', 'claims'), ('public', 'claim_evidence'), + ('public', 'claim_edges'), ('public', 'sources'), ('public', 'personas'), + ('public', 'strategies'), ('public', 'beliefs'), ('public', 'blindspots'), + ('public', 'agent_roles'), ('public', 'peer_models'), ('public', 'behavioral_rules'), + ('public', 'contributor_rules'), ('public', 'reasoning_tools'), + ('public', 'governance_gates'), ('kb_stage', 'kb_proposals') + ) + and relation.relkind <> 'r'; + if unexpected is not null then + raise exception 'Leo read allowlist contains a non-base relation: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s->%s', inheritance.inhparent::pg_catalog.regclass, inheritance.inhrelid::pg_catalog.regclass), + ', ' + ) + into unexpected + from pg_catalog.pg_inherits inheritance + where inheritance.inhparent in ( + 'public.agents'::pg_catalog.regclass, 'public.claims'::pg_catalog.regclass, + 'public.claim_evidence'::pg_catalog.regclass, 'public.claim_edges'::pg_catalog.regclass, + 'public.sources'::pg_catalog.regclass, 'public.personas'::pg_catalog.regclass, + 'public.strategies'::pg_catalog.regclass, 'public.beliefs'::pg_catalog.regclass, + 'public.blindspots'::pg_catalog.regclass, 'public.agent_roles'::pg_catalog.regclass, + 'public.peer_models'::pg_catalog.regclass, 'public.behavioral_rules'::pg_catalog.regclass, + 'public.contributor_rules'::pg_catalog.regclass, 'public.reasoning_tools'::pg_catalog.regclass, + 'public.governance_gates'::pg_catalog.regclass, 'kb_stage.kb_proposals'::pg_catalog.regclass + ) + or inheritance.inhrelid in ( + 'public.agents'::pg_catalog.regclass, 'public.claims'::pg_catalog.regclass, + 'public.claim_evidence'::pg_catalog.regclass, 'public.claim_edges'::pg_catalog.regclass, + 'public.sources'::pg_catalog.regclass, 'public.personas'::pg_catalog.regclass, + 'public.strategies'::pg_catalog.regclass, 'public.beliefs'::pg_catalog.regclass, + 'public.blindspots'::pg_catalog.regclass, 'public.agent_roles'::pg_catalog.regclass, + 'public.peer_models'::pg_catalog.regclass, 'public.behavioral_rules'::pg_catalog.regclass, + 'public.contributor_rules'::pg_catalog.regclass, 'public.reasoning_tools'::pg_catalog.regclass, + 'public.governance_gates'::pg_catalog.regclass, 'kb_stage.kb_proposals'::pg_catalog.regclass + ); + if unexpected is not null then + raise exception 'Leo read allowlist has inheritance edges: %', unexpected; + end if; + + with expected(ordinal, attname, typname, not_null, has_default, collated) as ( + values + (1, 'id', 'uuid', true, true, false), + (2, 'proposal_type', 'text', true, false, true), + (3, 'status', 'text', true, true, true), + (4, 'proposed_by_handle', 'text', false, false, true), + (5, 'proposed_by_agent_id', 'uuid', false, false, false), + (6, 'channel', 'text', true, true, true), + (7, 'source_ref', 'text', false, false, true), + (8, 'rationale', 'text', true, false, true), + (9, 'payload', 'jsonb', true, false, false), + (10, 'reviewed_by_handle', 'text', false, false, true), + (11, 'reviewed_by_agent_id', 'uuid', false, false, false), + (12, 'reviewed_at', 'timestamptz', false, false, false), + (13, 'review_note', 'text', false, false, true), + (14, 'applied_by_handle', 'text', false, false, true), + (15, 'applied_by_agent_id', 'uuid', false, false, false), + (16, 'applied_at', 'timestamptz', false, false, false), + (17, 'created_at', 'timestamptz', true, true, false), + (18, 'updated_at', 'timestamptz', true, true, false) + ), actual as ( + select attribute.attnum::int as ordinal, + attribute.attname, + type_row.typname, + attribute.attnotnull as not_null, + attribute.atthasdef as has_default, + attribute.attidentity, + attribute.attgenerated, + attribute.atttypmod, + type_namespace.nspname as type_namespace, + attribute.attcollation = 'pg_catalog.default'::pg_catalog.regcollation as collated + from pg_catalog.pg_attribute attribute + join pg_catalog.pg_type type_row on type_row.oid = attribute.atttypid + join pg_catalog.pg_namespace type_namespace on type_namespace.oid = type_row.typnamespace + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attnum > 0 + and not attribute.attisdropped + ) + select pg_catalog.string_agg(coalesce(expected.attname, actual.attname), ', ' order by coalesce(expected.ordinal, actual.ordinal)) + into unexpected + from expected + full join actual using (ordinal, attname) + where expected.ordinal is null + or actual.ordinal is null + or expected.typname is distinct from actual.typname + or expected.not_null is distinct from actual.not_null + or expected.has_default is distinct from actual.has_default + or expected.collated is distinct from actual.collated + or actual.attidentity <> '' + or actual.attgenerated <> '' + or actual.atttypmod <> -1 + or actual.type_namespace <> 'pg_catalog'; + if unexpected is not null then + raise exception 'kb_stage.kb_proposals column contract drifted: %', unexpected; + end if; + + if exists ( + select 1 + from pg_catalog.pg_trigger trigger_row + where trigger_row.tgrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and not trigger_row.tgisinternal + ) or exists ( + select 1 + from pg_catalog.pg_rewrite rewrite_row + where rewrite_row.ev_class = 'kb_stage.kb_proposals'::pg_catalog.regclass + ) or exists ( + select 1 + from pg_catalog.pg_policy policy_row + where policy_row.polrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + ) or exists ( + select 1 + from pg_catalog.pg_attribute attribute + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attnum > 0 + and not attribute.attisdropped + and attribute.attgenerated <> '' + ) then + raise exception 'kb_stage.kb_proposals has unsafe trigger/rule/RLS/generated topology'; + end if; + + with expected(attname, expression) as ( + values + ('id', 'gen_random_uuid()'), + ('status', '''pending_review''::text'), + ('channel', '''cli''::text'), + ('created_at', 'now()'), + ('updated_at', 'now()') + ), actual as ( + select attribute.attname, + pg_catalog.pg_get_expr(default_row.adbin, default_row.adrelid) as expression + from pg_catalog.pg_attribute attribute + join pg_catalog.pg_attrdef default_row + on default_row.adrelid = attribute.attrelid + and default_row.adnum = attribute.attnum + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attnum > 0 + and not attribute.attisdropped + ) + select pg_catalog.string_agg(coalesce(expected.attname, actual.attname), ', ') + into unexpected + from expected + full join actual using (attname) + where expected.expression is distinct from actual.expression; + if unexpected is not null then + raise exception 'kb_stage.kb_proposals default topology drifted: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s:%s', dependency.refclassid::pg_catalog.regclass, dependency.refobjid), + ', ' + ) + into unexpected + from pg_catalog.pg_attrdef default_row + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + where default_row.adrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and ( + dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + and dependency.refobjid not in ( + 'pg_catalog.gen_random_uuid()'::pg_catalog.regprocedure, + 'pg_catalog.now()'::pg_catalog.regprocedure + ) + or dependency.refclassid = 'pg_catalog.pg_type'::pg_catalog.regclass + and not exists ( + select 1 + from pg_catalog.pg_type type_row + join pg_catalog.pg_namespace namespace on namespace.oid = type_row.typnamespace + where type_row.oid = dependency.refobjid + and namespace.nspname = 'pg_catalog' + ) + or dependency.refclassid = 'pg_catalog.pg_operator'::pg_catalog.regclass + and not exists ( + select 1 + from pg_catalog.pg_operator operator_row + join pg_catalog.pg_namespace namespace on namespace.oid = operator_row.oprnamespace + where operator_row.oid = dependency.refobjid + and namespace.nspname = 'pg_catalog' + ) + or dependency.refclassid = 'pg_catalog.pg_collation'::pg_catalog.regclass + and not exists ( + select 1 + from pg_catalog.pg_collation collation_row + join pg_catalog.pg_namespace namespace on namespace.oid = collation_row.collnamespace + where collation_row.oid = dependency.refobjid + and namespace.nspname = 'pg_catalog' + ) + ); + if unexpected is not null then + raise exception 'kb_stage.kb_proposals defaults depend on unreviewed objects: %', unexpected; + end if; + + if ( + select pg_catalog.count(*) <> 6 + or pg_catalog.count(*) filter ( + where (constraint_row.conname, constraint_row.contype) in ( + ('kb_proposals_pkey', 'p'), + ('kb_proposals_proposal_type_check', 'c'), + ('kb_proposals_status_check', 'c'), + ('kb_proposals_applied_by_agent_id_fkey', 'f'), + ('kb_proposals_proposed_by_agent_id_fkey', 'f'), + ('kb_proposals_reviewed_by_agent_id_fkey', 'f') + ) + ) <> 6 + from pg_catalog.pg_constraint constraint_row + where constraint_row.conrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + ) or exists ( + select 1 + from pg_catalog.pg_constraint constraint_row + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_constraint'::pg_catalog.regclass + and dependency.objid = constraint_row.oid + and dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + join pg_catalog.pg_proc function_row on function_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where constraint_row.conrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and namespace.nspname <> 'pg_catalog' + ) or exists ( + select 1 + from pg_catalog.pg_index index_row + where index_row.indrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and ( + index_row.indexprs is not null + or ( + index_row.indpred is not null + and pg_catalog.pg_get_expr(index_row.indpred, index_row.indrelid) + <> '(proposed_by_handle IS NOT NULL)' + ) + ) + ) then + raise exception 'kb_stage.kb_proposals constraint/index topology drifted'; + end if; + select pg_catalog.string_agg( pg_catalog.format( '%I:%s:%s:grantable=%s', @@ -330,8 +804,7 @@ begin ) ) as acl join pg_catalog.pg_roles grantee on grantee.oid = acl.grantee - where database_row.datname in ('teleo_canonical', 'teleo_kb') - and grantee.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + where grantee.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') and not ( database_row.datname = 'teleo_canonical' and grantee.rolname = 'leoclean_kb_runtime' @@ -342,11 +815,86 @@ begin raise exception 'scoped Leo roles retain unexpected direct database ACLs: %', unexpected; end if; - if has_database_privilege('leoclean_kb_runtime', 'teleo_canonical', 'CREATE') - or has_database_privilege('leoclean_kb_runtime', 'teleo_kb', 'CREATE') - or has_database_privilege('leoclean_kb_stage_owner', 'teleo_canonical', 'CREATE') - or has_database_privilege('leoclean_kb_stage_owner', 'teleo_kb', 'CREATE') then - raise exception 'a scoped Leo role unexpectedly has database CREATE'; + select pg_catalog.string_agg(database_row.datname, ', ' order by database_row.datname) + into unexpected + from pg_catalog.pg_database database_row + where database_row.datname <> 'teleo_canonical' + and has_database_privilege('leoclean_kb_runtime', database_row.oid, 'CONNECT'); + if unexpected is not null then + raise exception 'runtime CONNECT reaches a noncanonical database: %', unexpected; + end if; + + select pg_catalog.string_agg(database_row.datname, ', ' order by database_row.datname) + into unexpected + from pg_catalog.pg_database database_row + where has_database_privilege('leoclean_kb_stage_owner', database_row.oid, 'CONNECT'); + if unexpected is not null then + raise exception 'stage owner CONNECT reaches a database: %', unexpected; + end if; + + select pg_catalog.string_agg(database_row.datname, ', ' order by database_row.datname) + into unexpected + from pg_catalog.pg_database database_row + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where acl.grantee = 0 + and acl.privilege_type = 'CONNECT'; + if unexpected is not null then + raise exception 'PUBLIC retains database CONNECT: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s:%s', setting_row.setdatabase, setting_row.setconfig), + ', ' + ) + into unexpected + from pg_catalog.pg_db_role_setting setting_row + where setting_row.setrole in (runtime_oid, owner_oid) + and not ( + setting_row.setrole = runtime_oid + and setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = 'teleo_canonical' + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ); + if unexpected is not null or not exists ( + select 1 + from pg_catalog.pg_db_role_setting setting_row + where setting_row.setrole = runtime_oid + and setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = 'teleo_canonical' + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ) then + raise exception 'scoped Leo role settings differ from the reviewed contract'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I:%s', database_row.datname, scoped_role.name), + ', ' + order by database_row.datname, scoped_role.name + ) + into unexpected + from pg_catalog.pg_database database_row + cross join (values ('leoclean_kb_runtime'), ('leoclean_kb_stage_owner')) scoped_role(name) + where has_database_privilege(scoped_role.name, database_row.oid, 'CREATE'); + if unexpected is not null then + raise exception 'a scoped Leo role unexpectedly has database CREATE: %', unexpected; end if; select pg_catalog.string_agg( @@ -382,6 +930,75 @@ begin raise exception 'leoclean_kb_stage_owner owns unexpected database objects: %', unexpected; end if; + select pg_catalog.string_agg(metadata.oid::text, ', ' order by metadata.oid) + into unexpected + from pg_catalog.pg_largeobject_metadata metadata + where metadata.lomowner in (runtime_oid, owner_oid) + or exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(metadata.lomacl, pg_catalog.acldefault('L', metadata.lomowner)) + ) acl + where acl.grantee in (0, runtime_oid, owner_oid) + and acl.privilege_type in ('SELECT', 'UPDATE') + ); + if unexpected is not null then + raise exception 'scoped Leo roles retain large-object ownership or ACL reachability: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I(%s)', function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid)), + ', ' + ) + into unexpected + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and ( + has_function_privilege('leoclean_kb_runtime', function_row.oid, 'EXECUTE') + or has_function_privilege('leoclean_kb_stage_owner', function_row.oid, 'EXECUTE') + ); + if unexpected is not null then + raise exception 'PUBLIC or inherited large-object mutators remain executable: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I(%s)', function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid)), + ', ' + order by function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid) + ) + into unexpected + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + cross join lateral pg_catalog.aclexplode( + coalesce(function_row.proacl, pg_catalog.acldefault('f', function_row.proowner)) + ) acl + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and acl.grantee = 0 + and acl.privilege_type = 'EXECUTE'; + if unexpected is not null then + raise exception 'PUBLIC retains large-object mutator EXECUTE: %', unexpected; + end if; + + select pg_catalog.string_agg(parameter_acl.parname, ', ' order by parameter_acl.parname) + into unexpected + from pg_catalog.pg_parameter_acl parameter_acl + where has_parameter_privilege('leoclean_kb_runtime', parameter_acl.parname, 'SET') + or has_parameter_privilege('leoclean_kb_runtime', parameter_acl.parname, 'ALTER SYSTEM') + or has_parameter_privilege('leoclean_kb_stage_owner', parameter_acl.parname, 'SET') + or has_parameter_privilege('leoclean_kb_stage_owner', parameter_acl.parname, 'ALTER SYSTEM'); + if unexpected is not null then + raise exception 'scoped Leo roles retain parameter privileges: %', unexpected; + end if; + if has_schema_privilege('leoclean_kb_runtime', 'public', 'CREATE') or has_schema_privilege('leoclean_kb_runtime', 'kb_stage', 'CREATE') or has_schema_privilege('leoclean_kb_stage_owner', 'public', 'CREATE') @@ -698,137 +1315,9 @@ begin end $verification$; -commit; - --- The legacy restore database is not an audit fallback for the scoped runtime. --- Strip direct ACLs from both scoped roles and fail if PUBLIC/group grants still --- make any teleo_restore relation reachable. -\connect teleo_kb - -begin; - -select format('revoke all on schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) - from pg_catalog.pg_namespace namespace - where namespace.nspname = 'teleo_restore' -\gexec - -select format('revoke all privileges on all tables in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) - from pg_catalog.pg_namespace namespace - where namespace.nspname = 'teleo_restore' -\gexec - -select format('revoke all privileges on all sequences in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) - from pg_catalog.pg_namespace namespace - where namespace.nspname = 'teleo_restore' -\gexec - -select format('revoke all privileges on all functions in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) - from pg_catalog.pg_namespace namespace - where namespace.nspname = 'teleo_restore' -\gexec - -select format('revoke all privileges on all procedures in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) - from pg_catalog.pg_namespace namespace - where namespace.nspname = 'teleo_restore' -\gexec - -select format( - 'revoke all privileges (%s) on table %I.%I from %I', - pg_catalog.string_agg(pg_catalog.quote_ident(attribute.attname), ', ' order by attribute.attnum), - namespace.nspname, - relation.relname, - target.rolname - ) - from pg_catalog.pg_class relation - join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace - join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid - cross join ( - values ('leoclean_kb_runtime'), ('leoclean_kb_stage_owner') - ) as target(rolname) - where namespace.nspname = 'teleo_restore' - and relation.relkind in ('r', 'p', 'v', 'm', 'f') - and attribute.attnum > 0 - and not attribute.attisdropped - group by namespace.nspname, relation.relname, target.rolname -\gexec - -do $audit_verification$ -declare - unexpected text; - scoped_role text; -begin - foreach scoped_role in array array['leoclean_kb_runtime', 'leoclean_kb_stage_owner'] loop - if to_regnamespace('teleo_restore') is not null - and (has_schema_privilege(scoped_role, 'teleo_restore', 'USAGE') - or has_schema_privilege(scoped_role, 'teleo_restore', 'CREATE')) then - raise exception '% unexpectedly has teleo_restore schema access', scoped_role; - end if; - - select pg_catalog.string_agg( - pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name), - ', ' - order by relation.relname, privilege.name - ) - into unexpected - from pg_catalog.pg_class relation - join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace - cross join ( - values ('SELECT'), ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER') - ) privilege(name) - where namespace.nspname = 'teleo_restore' - and relation.relkind in ('r', 'p', 'v', 'm', 'f') - and has_table_privilege(scoped_role, relation.oid, privilege.name); - if unexpected is not null then - raise exception '% unexpectedly has teleo_restore table access: %', scoped_role, unexpected; - end if; - - select pg_catalog.string_agg( - pg_catalog.format('%I.%I.%I:%s', namespace.nspname, relation.relname, attribute.attname, privilege.name), - ', ' - order by relation.relname, attribute.attnum, privilege.name - ) - into unexpected - from pg_catalog.pg_class relation - join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace - join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid - cross join (values ('SELECT'), ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) - where namespace.nspname = 'teleo_restore' - and relation.relkind in ('r', 'p', 'v', 'm', 'f') - and attribute.attnum > 0 - and not attribute.attisdropped - and has_column_privilege(scoped_role, relation.oid, attribute.attnum, privilege.name); - if unexpected is not null then - raise exception '% unexpectedly has teleo_restore column access: %', scoped_role, unexpected; - end if; - - select pg_catalog.string_agg( - pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name), - ', ' - order by relation.relname, privilege.name - ) - into unexpected - from pg_catalog.pg_class relation - join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace - cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) - where namespace.nspname = 'teleo_restore' - and relation.relkind = 'S' - and has_sequence_privilege(scoped_role, relation.oid, privilege.name); - if unexpected is not null then - raise exception '% unexpectedly has teleo_restore sequence access: %', scoped_role, unexpected; - end if; - end loop; -end -$audit_verification$; - -commit; - -\connect teleo_canonical - --- LOGIN is the last state change. If this verification fails, the transaction --- rolls back and the role remains NOLOGIN while prior least-privilege grants --- stay inert for direct authentication. -begin; - +-- LOGIN is the last state change in the same transaction as every database ACL, +-- canonical privilege, large-object, and topology assertion. Any failure rolls +-- back the complete privilege migration and leaves the role NOLOGIN. alter role leoclean_kb_runtime with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; @@ -838,14 +1327,16 @@ begin select 1 from pg_catalog.pg_roles role_row where role_row.rolname = 'leoclean_kb_runtime' - and (not role_row.rolcanlogin - or role_row.rolsuper - or role_row.rolcreatedb - or role_row.rolcreaterole - or role_row.rolinherit - or role_row.rolreplication - or role_row.rolbypassrls - or role_row.rolconnlimit <> 8) + and ( + not role_row.rolcanlogin + or role_row.rolsuper + or role_row.rolcreatedb + or role_row.rolcreaterole + or role_row.rolinherit + or role_row.rolreplication + or role_row.rolbypassrls + or role_row.rolconnlimit <> 8 + ) ) then raise exception 'leoclean_kb_runtime final login attributes are unsafe'; end if; @@ -853,32 +1344,3 @@ end $login_verification$; commit; - -select jsonb_build_object( - 'role', rolname, - 'superuser', rolsuper, - 'create_db', rolcreatedb, - 'create_role', rolcreaterole, - 'inherit', rolinherit, - 'replication', rolreplication, - 'bypass_rls', rolbypassrls, - 'has_role_memberships', exists ( - select 1 - from pg_catalog.pg_auth_members membership - where membership.roleid = pg_catalog.pg_roles.oid - or membership.member = pg_catalog.pg_roles.oid - ), - 'can_select_claims', has_table_privilege(rolname, 'public.claims', 'SELECT'), - 'can_select_proposals', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'SELECT'), - 'can_insert_proposals_directly', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'INSERT'), - 'can_connect_legacy_database', has_database_privilege(rolname, 'teleo_kb', 'CONNECT'), - 'can_create_database_objects', has_database_privilege(rolname, 'teleo_canonical', 'CREATE'), - 'can_create_temp_objects', has_database_privilege(rolname, 'teleo_canonical', 'TEMP'), - 'can_stage_proposal', has_function_privilege( - rolname, - 'kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)', - 'EXECUTE' - ) -)::text -from pg_catalog.pg_roles -where rolname = 'leoclean_kb_runtime'; diff --git a/ops/provision_gcp_leoclean_runtime_role.sh b/ops/provision_gcp_leoclean_runtime_role.sh new file mode 100755 index 0000000..9ff89f8 --- /dev/null +++ b/ops/provision_gcp_leoclean_runtime_role.sh @@ -0,0 +1,150 @@ +#!/bin/bash +# Provision the scoped GCP staging role without placing its cleartext password +# in SQL text, argv, command history, or PostgreSQL statement logs. +set +x +set -euo pipefail +umask 077 +ulimit -c 0 + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PATH +runtime_password=${TELEO_LEOCLEAN_DB_PASSWORD-} +administrator_password=${PGPASSWORD-} +unset TELEO_LEOCLEAN_DB_PASSWORD PGPASSWORD + +script_path=${BASH_SOURCE[0]} +case "$script_path" in + */*) script_directory=${script_path%/*} ;; + *) script_directory=. ;; +esac +SCRIPT_DIR="$(cd -- "$script_directory" && pwd -P)" +SQL_FILE="$SCRIPT_DIR/gcp_leoclean_runtime_role.sql" +SERVER_CA_FILE="$SCRIPT_DIR/gcp-teleo-pgvector-standby-server-ca.pem" +PSQL_BIN=/usr/bin/psql +SETSID_BIN=/usr/bin/setsid +ENV_BIN=/usr/bin/env +SHA256SUM_BIN=/usr/bin/sha256sum +SERVER_CA_SHA256=80701e768f0e1f6b9d621aa0b53f6e851daaa276c6d9a8e51a300fbc015539cb + +assert_root_owned_nonwritable() { + local target="$1" state uid mode + state="$(stat -Lc '%u:%a' -- "$target")" + uid=${state%%:*} + mode=${state#*:} + if [ "$uid" != 0 ] || (( (8#$mode & 8#022) != 0 )); then + echo "Refusing an untrusted PostgreSQL executable path." >&2 + return 1 + fi +} + +assert_trusted_executable() { + local requested="$1" resolved current + [ -x "$requested" ] || { + echo "Required PostgreSQL client is unavailable." >&2 + return 1 + } + resolved="$(readlink -f -- "$requested")" + [ -n "$resolved" ] && [ -f "$resolved" ] && [ -x "$resolved" ] || { + echo "Required PostgreSQL client did not resolve to an executable file." >&2 + return 1 + } + assert_root_owned_nonwritable "$requested" + current="$(dirname "$requested")" + while :; do + assert_root_owned_nonwritable "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done + current="$resolved" + while :; do + assert_root_owned_nonwritable "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done +} + +if [ ! -f "$SQL_FILE" ] || [ -L "$SQL_FILE" ] || [ ! -f "$SERVER_CA_FILE" ] || [ -L "$SERVER_CA_FILE" ]; then + echo "Reviewed GCP runtime-role SQL is unavailable." >&2 + exit 66 +fi +assert_trusted_executable "$PSQL_BIN" +assert_trusted_executable "$SETSID_BIN" +assert_trusted_executable "$ENV_BIN" +assert_trusted_executable "$SHA256SUM_BIN" +if [ "$("$SHA256SUM_BIN" "$SERVER_CA_FILE")" != "$SERVER_CA_SHA256 $SERVER_CA_FILE" ]; then + echo "Reviewed Cloud SQL server CA does not match the pinned certificate." >&2 + exit 66 +fi + +if [ -z "$runtime_password" ] || [ "${#runtime_password}" -gt 4096 ]; then + echo "TELEO_LEOCLEAN_DB_PASSWORD must contain a bounded nonempty value." >&2 + exit 64 +fi +case "$runtime_password" in + *$'\n'*|*$'\r'*) + echo "TELEO_LEOCLEAN_DB_PASSWORD must not contain line breaks." >&2 + exit 64 + ;; +esac +if [ -z "$administrator_password" ] || [ "${#administrator_password}" -gt 4096 ]; then + echo "PGPASSWORD must contain a bounded nonempty administrator credential." >&2 + exit 64 +fi +case "$administrator_password" in + *$'\n'*|*$'\r'*) + echo "PGPASSWORD must not contain line breaks." >&2 + exit 64 + ;; +esac +cleanup_password() { + runtime_password= + administrator_password= + unset runtime_password administrator_password +} +provision_pid= +terminate_provisioning() { + local exit_status="$1" + trap - INT TERM + if [[ "$provision_pid" =~ ^[1-9][0-9]*$ ]]; then + kill -TERM -- "-$provision_pid" 2>/dev/null || kill -TERM -- "$provision_pid" 2>/dev/null || true + wait "$provision_pid" 2>/dev/null || true + provision_pid= + fi + cleanup_password + exit "$exit_status" +} +trap cleanup_password EXIT +trap 'terminate_provisioning 130' INT +trap 'terminate_provisioning 143' TERM + +# \password encrypts client-side. --no-password ensures an administrator-login +# prompt cannot consume either of the two runtime-password records from stdin. +# setsid detaches psql from any controlling terminal without --fork, keeping +# its PID as the session/process-group leader so an interruption can terminate +# and reap the complete credential-bearing child before this wrapper returns. +{ + printf '%s\n' "$runtime_password" + printf '%s\n' "$runtime_password" +} | "$ENV_BIN" -i \ + PATH="$PATH" \ + LANG=C LC_ALL=C \ + PYTHONNOUSERSITE=1 PYTHONSAFEPATH=1 \ + PGHOST=10.61.0.3 \ + PGPORT=5432 \ + PGDATABASE=teleo_canonical \ + PGUSER=postgres \ + PGPASSWORD="$administrator_password" \ + PGSSLMODE=verify-ca \ + PGSSLROOTCERT="$SERVER_CA_FILE" \ + "$SETSID_BIN" -- "$PSQL_BIN" \ + --no-psqlrc \ + --no-password \ + --set=ON_ERROR_STOP=1 \ + --file="$SQL_FILE" & +provision_pid=$! +set +e +wait "$provision_pid" +status=$? +set -e +provision_pid= +exit "$status" diff --git a/ops/verify_gcp_leoclean_runtime_permissions.py b/ops/verify_gcp_leoclean_runtime_permissions.py index bdd44dd..94ccc4f 100644 --- a/ops/verify_gcp_leoclean_runtime_permissions.py +++ b/ops/verify_gcp_leoclean_runtime_permissions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 -I """Retain a sanitized least-privilege receipt for the GCP Leo runtime role. This verifier is intentionally service-independent. It must run on the private @@ -19,8 +19,8 @@ import re import stat import subprocess import sys -import tempfile from collections.abc import Callable, Mapping +from contextlib import nullcontext from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -29,8 +29,10 @@ from typing import Any PROJECT_ID = "teleo-501523" PRIVATE_CLOUDSQL_HOST = "10.61.0.3" PRIVATE_CLOUDSQL_PORT = 5432 +EXPECTED_SYSTEM_IDENTIFIER = "7659718422914359312" CANONICAL_DATABASE = "teleo_canonical" LEGACY_DATABASE = "teleo_kb" +TEMPLATE_DATABASE = "template1" RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime" STAGE_OWNER_DATABASE_ROLE = "leoclean_kb_stage_owner" RUNTIME_UNIX_USER = "teleo" @@ -38,6 +40,9 @@ SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-passwor ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password" GCLOUD_BIN = "/usr/bin/gcloud" PSQL_BIN = "/usr/bin/psql" +SERVER_CA_PATH = Path("/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem") +RUNTIME_CLOUDSDK_CONFIG = Path("/usr/local/libexec/livingip/leoclean-kb/gcloud-config") +SERVER_CA_SHA256 = "80701e768f0e1f6b9d621aa0b53f6e851daaa276c6d9a8e51a300fbc015539cb" COMMAND_TIMEOUT_SECONDS = 30 CHILD_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" RUN_ID_RE = re.compile(r"[a-z0-9][a-z0-9-]{6,62}[a-z0-9]\Z") @@ -195,8 +200,14 @@ CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( "column_dml_grants", "database_create_grants", "missing_allowed_table_selects", + "large_object_acl_privileges", + "large_object_mutation_routine_execute", + "public_large_object_mutation_routine_execute", + "other_scoped_backends", "owned_database_objects", + "owned_large_objects", "owner_database_create_grants", + "owner_database_connect_grants", "owner_direct_database_acl_entries", "owner_missing_agent_select_columns", "owner_missing_proposal_insert_columns", @@ -210,38 +221,44 @@ CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( "owner_unexpected_schema_usage", "owner_unexpected_table_selects", "schema_create_grants", + "parameter_privileges", + "public_database_connect_grants", + "scoped_prepared_xacts", "sequence_grants", "table_dml_grants", "unexpected_column_selects", "unexpected_direct_database_acl_entries", + "unexpected_connectable_databases", "unexpected_routine_execute", + "unexpected_role_settings", "unexpected_schema_usage", "unexpected_security_definer_execute", "unexpected_table_selects", + "unsafe_allowed_relation_kinds", + "unsafe_allowed_relation_inheritance", + "unsafe_proposal_column_contract", + "unsafe_proposal_default_dependencies", + "unsafe_proposal_defaults", + "unsafe_proposal_constraint_dependencies", + "unsafe_proposal_index_expressions", + "unsafe_proposal_policies", + "unsafe_proposal_rewrite_rules", + "unsafe_proposal_triggers", + "unsafe_proposal_generated_columns", ) CATALOG_TRUE_FIELDS: tuple[str, ...] = ( "canonical_connect", "owner_grant_contract", "owner_schema_usage", + "proposal_table_contract", + "proposal_constraint_shape", "runtime_connect_acl_exact", "runtime_schema_usage", + "runtime_setting_contract", + "effective_setting_contract", ) -LEGACY_CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( - "owner_column_privileges", - "owner_routine_privileges", - "owner_schema_privileges", - "owner_sequence_privileges", - "owner_table_privileges", - "runtime_column_privileges", - "runtime_routine_privileges", - "runtime_schema_privileges", - "runtime_sequence_privileges", - "runtime_table_privileges", -) - - @dataclass(frozen=True) class CommandResult: returncode: int @@ -250,6 +267,7 @@ class CommandResult: Runner = Callable[[list[str], Mapping[str, str], int, bool], CommandResult] +DependencyValidator = Callable[[], None] @dataclass(frozen=True) @@ -258,6 +276,7 @@ class NegativeCheck: database: str sql: str expected_sqlstate: str + expected_connection_denial: bool = False class VerificationError(RuntimeError): @@ -328,14 +347,110 @@ def _base_child_environment(_source: Mapping[str, str]) -> dict[str, str]: "LANG": "C", "LC_ALL": "C", "PATH": CHILD_PATH, + "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", } +def _path_and_parents(path: Path) -> tuple[Path, ...]: + resolved = path if path.is_absolute() else path.absolute() + return (resolved, *resolved.parents) + + +def _assert_root_owned_nonwritable(path: Path, *, follow_symlinks: bool = True) -> None: + try: + metadata = path.stat() if follow_symlinks else path.lstat() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if metadata.st_uid != 0 or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + if os.geteuid() != 0 and os.access(path, os.W_OK): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + + +def _assert_trusted_executable(raw_path: str) -> None: + requested = Path(raw_path) + if not requested.is_absolute(): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + try: + resolved = requested.resolve(strict=True) + metadata = resolved.stat() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if not stat.S_ISREG(metadata.st_mode) or not os.access(resolved, os.X_OK): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + try: + requested_metadata = requested.lstat() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if requested_metadata.st_uid != 0 or ( + not stat.S_ISLNK(requested_metadata.st_mode) + and requested_metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + ): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + for candidate in (*_path_and_parents(requested.parent), *_path_and_parents(resolved)): + _assert_root_owned_nonwritable(candidate) + + +def validate_runtime_dependencies() -> None: + _assert_trusted_executable(GCLOUD_BIN) + _assert_trusted_executable(PSQL_BIN) + runtime_cloudsdk_config_path() + server_ca = runtime_server_ca_path() + for candidate in _path_and_parents(server_ca): + _assert_root_owned_nonwritable(candidate) + try: + certificate = server_ca.read_bytes() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if hashlib.sha256(certificate).hexdigest() != SERVER_CA_SHA256: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + + +def runtime_server_ca_path() -> Path: + raw = os.environ.get("TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT", str(SERVER_CA_PATH)) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if resolved != candidate or not resolved.is_file(): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + return resolved + + +def selected_server_ca_path() -> Path: + return Path(os.environ.get("TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT", str(SERVER_CA_PATH))) + + +def runtime_cloudsdk_config_path() -> Path: + raw = os.environ.get("TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG", str(RUNTIME_CLOUDSDK_CONFIG)) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + if resolved != candidate or not resolved.is_dir() or any(resolved.iterdir()): + raise OSError + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + for part in _path_and_parents(resolved): + _assert_root_owned_nonwritable(part) + return resolved + + +def selected_cloudsdk_config_path() -> Path: + return Path(os.environ.get("TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG", str(RUNTIME_CLOUDSDK_CONFIG))) + + def _gcloud_environment(source: Mapping[str, str], config_dir: Path) -> dict[str, str]: env = _base_child_environment(source) env.update( { "CLOUDSDK_CONFIG": str(config_dir), + "CLOUDSDK_CORE_DISABLE_FILE_LOGGING": "true", "CLOUDSDK_CORE_DISABLE_PROMPTS": "1", "CLOUDSDK_CORE_PROJECT": PROJECT_ID, } @@ -345,7 +460,13 @@ def _gcloud_environment(source: Mapping[str, str], config_dir: Path) -> dict[str def _psql_environment(source: Mapping[str, str], password: str) -> dict[str, str]: env = _base_child_environment(source) - env["PGPASSWORD"] = password + env.update( + { + "PGPASSWORD": password, + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": str(selected_server_ca_path()), + } + ) return env @@ -424,7 +545,7 @@ def _assert_administrator_secret_denied(*, env: Mapping[str, str], runner: Runne def _conninfo(database: str) -> str: return ( f"host={PRIVATE_CLOUDSQL_HOST} port={PRIVATE_CLOUDSQL_PORT} dbname={database} " - f"user={RUNTIME_DATABASE_ROLE} sslmode=require connect_timeout=10 " + f"user={RUNTIME_DATABASE_ROLE} sslmode=verify-ca sslrootcert={selected_server_ca_path()} connect_timeout=10 " "application_name=leoclean_runtime_permission_verifier" ) @@ -515,7 +636,8 @@ select pg_catalog.jsonb_build_object( 'ssl_version', coalesce( (select version from pg_catalog.pg_stat_ssl where pid = pg_catalog.pg_backend_pid()), '' - ) + ), + 'system_identifier', (select system_identifier::text from pg_catalog.pg_control_system()) )::text; """.strip() @@ -689,17 +811,60 @@ with runtime_role as ( ), allowed_relation as ( select allowed_select.nspname, allowed_select.relname, - relation.oid + relation.oid, + relation.relkind from allowed_select left join pg_catalog.pg_namespace namespace on namespace.nspname = allowed_select.nspname left join pg_catalog.pg_class relation on relation.relnamespace = namespace.oid and relation.relname = allowed_select.relname - and relation.relkind in ('r', 'p', 'v', 'm', 'f') ), owner_insert_column(attname) as ( values {owner_insert_values} +), expected_proposal_column(ordinal, attname, typname, not_null, has_default, collated) as ( + values + (1, 'id', 'uuid', true, true, false), + (2, 'proposal_type', 'text', true, false, true), + (3, 'status', 'text', true, true, true), + (4, 'proposed_by_handle', 'text', false, false, true), + (5, 'proposed_by_agent_id', 'uuid', false, false, false), + (6, 'channel', 'text', true, true, true), + (7, 'source_ref', 'text', false, false, true), + (8, 'rationale', 'text', true, false, true), + (9, 'payload', 'jsonb', true, false, false), + (10, 'reviewed_by_handle', 'text', false, false, true), + (11, 'reviewed_by_agent_id', 'uuid', false, false, false), + (12, 'reviewed_at', 'timestamptz', false, false, false), + (13, 'review_note', 'text', false, false, true), + (14, 'applied_by_handle', 'text', false, false, true), + (15, 'applied_by_agent_id', 'uuid', false, false, false), + (16, 'applied_at', 'timestamptz', false, false, false), + (17, 'created_at', 'timestamptz', true, true, false), + (18, 'updated_at', 'timestamptz', true, true, false) +), proposal_table as ( + select relation.* + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' +), expected_proposal_default(attname, expression) as ( + values + ('id', 'gen_random_uuid()'), + ('status', '''pending_review''::text'), + ('channel', '''cli''::text'), + ('created_at', 'now()'), + ('updated_at', 'now()') +), actual_proposal_default as ( + select attribute.attname, + pg_catalog.pg_get_expr(default_row.adbin, default_row.adrelid) as expression + from proposal_table + join pg_catalog.pg_attribute attribute on attribute.attrelid = proposal_table.oid + join pg_catalog.pg_attrdef default_row + on default_row.adrelid = attribute.attrelid + and default_row.adnum = attribute.attnum + where attribute.attnum > 0 + and not attribute.attisdropped ) select pg_catalog.jsonb_build_object( 'canonical_connect', coalesce( @@ -720,6 +885,75 @@ select pg_catalog.jsonb_build_object( where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) and acl.grantee = runtime_role.oid ), false), + 'runtime_setting_contract', coalesce(( + select pg_catalog.count(*) = 1 + and pg_catalog.bool_and( + setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ) + from pg_catalog.pg_db_role_setting setting_row + join runtime_role on runtime_role.oid = setting_row.setrole + ), false), + 'unexpected_role_settings', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_db_role_setting setting_row + where setting_row.setrole in ((select oid from runtime_role), (select oid from owner_role)) + and not ( + setting_row.setrole = (select oid from runtime_role) + and setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ) + ), + 'effective_setting_contract', ( + pg_catalog.current_setting('search_path') = 'pg_catalog, public, kb_stage' + and pg_catalog.current_setting('statement_timeout') = '15s' + and pg_catalog.current_setting('lock_timeout') = '2s' + and pg_catalog.current_setting('session_preload_libraries') = '' + and pg_catalog.current_setting('local_preload_libraries') = '' + and pg_catalog.current_setting('role') = 'none' + ), + 'unexpected_connectable_databases', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + where database_row.datname <> {_sql_literal(CANONICAL_DATABASE)} + and pg_catalog.has_database_privilege(current_user, database_row.oid, 'CONNECT') + ), + 'owner_database_connect_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + where pg_catalog.has_database_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + database_row.oid, + 'CONNECT' + ) + ), + 'public_database_connect_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where acl.grantee = 0 + and acl.privilege_type = 'CONNECT' + ), 'unexpected_direct_database_acl_entries', ( select pg_catalog.count(*)::int from pg_catalog.pg_database database_row @@ -727,8 +961,7 @@ select pg_catalog.jsonb_build_object( cross join lateral pg_catalog.aclexplode( coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) ) acl - where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) - and acl.grantee = runtime_role.oid + where acl.grantee = runtime_role.oid and not ( database_row.datname = {_sql_literal(CANONICAL_DATABASE)} and acl.privilege_type = 'CONNECT' @@ -742,20 +975,19 @@ select pg_catalog.jsonb_build_object( cross join lateral pg_catalog.aclexplode( coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) ) acl - where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) - and acl.grantee = owner_role.oid + where acl.grantee = owner_role.oid ), 'database_create_grants', ( select pg_catalog.count(*)::int - from (values ({_sql_literal(CANONICAL_DATABASE)}), ({_sql_literal(LEGACY_DATABASE)})) database_name(name) - where pg_catalog.has_database_privilege(current_user, database_name.name, 'CREATE') + from pg_catalog.pg_database database_row + where pg_catalog.has_database_privilege(current_user, database_row.oid, 'CREATE') ), 'owner_database_create_grants', ( select pg_catalog.count(*)::int - from (values ({_sql_literal(CANONICAL_DATABASE)}), ({_sql_literal(LEGACY_DATABASE)})) database_name(name) + from pg_catalog.pg_database database_row where pg_catalog.has_database_privilege( {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, - database_name.name, + database_row.oid, 'CREATE' ) ), @@ -837,6 +1069,241 @@ select pg_catalog.jsonb_build_object( false ) ), + 'unsafe_allowed_relation_kinds', ( + select pg_catalog.count(*)::int + from allowed_relation + where allowed_relation.relkind is distinct from 'r'::"char" + ), + 'unsafe_allowed_relation_inheritance', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_inherits inheritance + where inheritance.inhparent in (select oid from allowed_relation where oid is not null) + or inheritance.inhrelid in (select oid from allowed_relation where oid is not null) + ), + 'unsafe_proposal_column_contract', ( + select pg_catalog.count(*)::int + from expected_proposal_column expected + full join ( + select attribute.attnum::int as ordinal, + attribute.attname, + type_row.typname, + attribute.attnotnull as not_null, + attribute.atthasdef as has_default, + attribute.attidentity, + attribute.attgenerated, + attribute.atttypmod, + type_namespace.nspname as type_namespace, + attribute.attcollation = 'pg_catalog.default'::pg_catalog.regcollation as collated + from proposal_table + join pg_catalog.pg_attribute attribute on attribute.attrelid = proposal_table.oid + join pg_catalog.pg_type type_row on type_row.oid = attribute.atttypid + join pg_catalog.pg_namespace type_namespace on type_namespace.oid = type_row.typnamespace + where attribute.attnum > 0 + and not attribute.attisdropped + ) actual using (ordinal, attname) + where expected.ordinal is null + or actual.ordinal is null + or expected.typname is distinct from actual.typname + or expected.not_null is distinct from actual.not_null + or expected.has_default is distinct from actual.has_default + or expected.collated is distinct from actual.collated + or actual.attidentity <> '' + or actual.attgenerated <> '' + or actual.atttypmod <> -1 + or actual.type_namespace <> 'pg_catalog' + ), + 'proposal_table_contract', coalesce(( + select pg_catalog.count(*) = 1 + and pg_catalog.bool_and( + proposal_table.relkind = 'r' + and proposal_table.relpersistence = 'p' + and not proposal_table.relrowsecurity + and not proposal_table.relforcerowsecurity + ) + from proposal_table + ), false), + 'unsafe_proposal_triggers', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_trigger trigger_row + join proposal_table on proposal_table.oid = trigger_row.tgrelid + where not trigger_row.tgisinternal + ), + 'unsafe_proposal_rewrite_rules', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_rewrite rewrite_row + join proposal_table on proposal_table.oid = rewrite_row.ev_class + ), + 'unsafe_proposal_policies', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_policy policy_row + join proposal_table on proposal_table.oid = policy_row.polrelid + ), + 'unsafe_proposal_generated_columns', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_attribute attribute + join proposal_table on proposal_table.oid = attribute.attrelid + where attribute.attnum > 0 + and not attribute.attisdropped + and attribute.attgenerated <> '' + ), + 'unsafe_proposal_defaults', ( + select pg_catalog.count(*)::int + from expected_proposal_default expected + full join actual_proposal_default actual using (attname) + where expected.expression is distinct from actual.expression + ), + 'unsafe_proposal_default_dependencies', ( + select pg_catalog.count(*)::int + from ( + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + join pg_catalog.pg_proc function_row on function_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname <> 'pg_catalog' + or function_row.oid not in ( + 'pg_catalog.gen_random_uuid()'::pg_catalog.regprocedure, + 'pg_catalog.now()'::pg_catalog.regprocedure + ) + union all + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_type'::pg_catalog.regclass + join pg_catalog.pg_type type_row on type_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = type_row.typnamespace + where namespace.nspname <> 'pg_catalog' + union all + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_operator'::pg_catalog.regclass + join pg_catalog.pg_operator operator_row on operator_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = operator_row.oprnamespace + where namespace.nspname <> 'pg_catalog' + union all + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_collation'::pg_catalog.regclass + join pg_catalog.pg_collation collation_row on collation_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = collation_row.collnamespace + where namespace.nspname <> 'pg_catalog' + ) unsafe_dependency + ), + 'owned_large_objects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_largeobject_metadata metadata + where metadata.lomowner in ((select oid from runtime_role), (select oid from owner_role)) + ), + 'large_object_acl_privileges', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_largeobject_metadata metadata + where exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(metadata.lomacl, pg_catalog.acldefault('L', metadata.lomowner)) + ) acl + where acl.grantee in (0, (select oid from runtime_role), (select oid from owner_role)) + and acl.privilege_type in ('SELECT', 'UPDATE') + ) + ), + 'large_object_mutation_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + cross join (values ({_sql_literal(RUNTIME_DATABASE_ROLE)}), ({_sql_literal(STAGE_OWNER_DATABASE_ROLE)})) scoped_role(name) + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and pg_catalog.has_function_privilege(scoped_role.name, function_row.oid, 'EXECUTE') + ), + 'public_large_object_mutation_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + cross join lateral pg_catalog.aclexplode( + coalesce(function_row.proacl, pg_catalog.acldefault('f', function_row.proowner)) + ) acl + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and acl.grantee = 0 + and acl.privilege_type = 'EXECUTE' + ), + 'parameter_privileges', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_parameter_acl parameter_acl + cross join (values ({_sql_literal(RUNTIME_DATABASE_ROLE)}), ({_sql_literal(STAGE_OWNER_DATABASE_ROLE)})) scoped_role(name) + where pg_catalog.has_parameter_privilege(scoped_role.name, parameter_acl.parname, 'SET') + or pg_catalog.has_parameter_privilege(scoped_role.name, parameter_acl.parname, 'ALTER SYSTEM') + ), + 'proposal_constraint_shape', coalesce(( + select pg_catalog.count(*) = 6 + and pg_catalog.count(*) filter ( + where (constraint_row.conname, constraint_row.contype) in ( + ('kb_proposals_pkey', 'p'), + ('kb_proposals_proposal_type_check', 'c'), + ('kb_proposals_status_check', 'c'), + ('kb_proposals_applied_by_agent_id_fkey', 'f'), + ('kb_proposals_proposed_by_agent_id_fkey', 'f'), + ('kb_proposals_reviewed_by_agent_id_fkey', 'f') + ) + ) = 6 + from pg_catalog.pg_constraint constraint_row + join proposal_table on proposal_table.oid = constraint_row.conrelid + ), false), + 'unsafe_proposal_constraint_dependencies', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_constraint constraint_row + join proposal_table on proposal_table.oid = constraint_row.conrelid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_constraint'::pg_catalog.regclass + and dependency.objid = constraint_row.oid + and dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + join pg_catalog.pg_proc function_row on function_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname <> 'pg_catalog' + ), + 'unsafe_proposal_index_expressions', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_index index_row + join proposal_table on proposal_table.oid = index_row.indrelid + where index_row.indexprs is not null + or ( + index_row.indpred is not null + and pg_catalog.pg_get_expr(index_row.indpred, index_row.indrelid) + <> '(proposed_by_handle IS NOT NULL)' + ) + ), + 'other_scoped_backends', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_stat_activity activity + where activity.pid <> pg_catalog.pg_backend_pid() + and activity.usename in ({_sql_literal(RUNTIME_DATABASE_ROLE)}, {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}) + ), + 'scoped_prepared_xacts', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_prepared_xacts prepared + where prepared.owner in ({_sql_literal(RUNTIME_DATABASE_ROLE)}, {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}) + ), 'table_dml_grants', ( select pg_catalog.count(*)::int from pg_catalog.pg_class relation @@ -1083,99 +1550,6 @@ select pg_catalog.jsonb_build_object( """.strip() -def _legacy_catalog_privilege_posture_sql() -> str: - return f""" -with scoped_role(role_name, field_prefix) as ( - values - ({_sql_literal(RUNTIME_DATABASE_ROLE)}, 'runtime'), - ({_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, 'owner') -), restore_schema as ( - select oid - from pg_catalog.pg_namespace - where nspname = 'teleo_restore' -), posture as ( - select scoped_role.field_prefix, - ( - select pg_catalog.count(*)::int - from restore_schema - cross join (values ('USAGE'), ('CREATE')) privilege(name) - where pg_catalog.has_schema_privilege( - scoped_role.role_name, - restore_schema.oid, - privilege.name - ) - ) as schema_privileges, - ( - select pg_catalog.count(*)::int - from pg_catalog.pg_class relation - join restore_schema on restore_schema.oid = relation.relnamespace - cross join ( - values ('SELECT'), ('INSERT'), ('UPDATE'), ('DELETE'), - ('TRUNCATE'), ('REFERENCES'), ('TRIGGER') - ) privilege(name) - where relation.relkind in ('r', 'p', 'v', 'm', 'f') - and pg_catalog.has_table_privilege( - scoped_role.role_name, - relation.oid, - privilege.name - ) - ) as table_privileges, - ( - select pg_catalog.count(*)::int - from pg_catalog.pg_class relation - join restore_schema on restore_schema.oid = relation.relnamespace - join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid - cross join (values ('SELECT'), ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) - where relation.relkind in ('r', 'p', 'v', 'm', 'f') - and attribute.attnum > 0 - and not attribute.attisdropped - and pg_catalog.has_column_privilege( - scoped_role.role_name, - relation.oid, - attribute.attnum, - privilege.name - ) - ) as column_privileges, - ( - select pg_catalog.count(*)::int - from pg_catalog.pg_class relation - join restore_schema on restore_schema.oid = relation.relnamespace - cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) - where relation.relkind = 'S' - and pg_catalog.has_sequence_privilege( - scoped_role.role_name, - relation.oid, - privilege.name - ) - ) as sequence_privileges, - ( - select pg_catalog.count(*)::int - from pg_catalog.pg_proc function_row - join restore_schema on restore_schema.oid = function_row.pronamespace - where pg_catalog.has_function_privilege( - scoped_role.role_name, - function_row.oid, - 'EXECUTE' - ) - ) as routine_privileges - from scoped_role -) -select pg_catalog.jsonb_build_object( - 'runtime_schema_privileges', max(schema_privileges) filter (where field_prefix = 'runtime'), - 'runtime_table_privileges', max(table_privileges) filter (where field_prefix = 'runtime'), - 'runtime_column_privileges', max(column_privileges) filter (where field_prefix = 'runtime'), - 'runtime_sequence_privileges', max(sequence_privileges) filter (where field_prefix = 'runtime'), - 'runtime_routine_privileges', max(routine_privileges) filter (where field_prefix = 'runtime'), - 'owner_schema_privileges', max(schema_privileges) filter (where field_prefix = 'owner'), - 'owner_table_privileges', max(table_privileges) filter (where field_prefix = 'owner'), - 'owner_column_privileges', max(column_privileges) filter (where field_prefix = 'owner'), - 'owner_sequence_privileges', max(sequence_privileges) filter (where field_prefix = 'owner'), - 'owner_routine_privileges', max(routine_privileges) filter (where field_prefix = 'owner') -)::text -from posture; -""".strip() - - def _canary_count_sql(source_ref: str) -> str: return f"select pg_catalog.count(*)::text from kb_stage.kb_proposals where source_ref = {_sql_literal(source_ref)};" @@ -1202,6 +1576,7 @@ select pg_catalog.jsonb_build_object( 'proposal', proposal )::text from staged; +set constraints all immediate; rollback; """.strip() @@ -1249,6 +1624,24 @@ def _negative_checks(run_id: str, source_ref: str) -> tuple[NegativeCheck, ...]: _transactional("delete from public.claims where false"), "42501", ), + NegativeCheck( + "large_object_creat", + CANONICAL_DATABASE, + _transactional("select pg_catalog.lo_creat(0)"), + "42501", + ), + NegativeCheck( + "large_object_create", + CANONICAL_DATABASE, + _transactional("select pg_catalog.lo_create(0)"), + "42501", + ), + NegativeCheck( + "large_object_from_bytea", + CANONICAL_DATABASE, + _transactional("select pg_catalog.lo_from_bytea(0, ''::bytea)"), + "42501", + ), NegativeCheck( "forged_proposer_overload", CANONICAL_DATABASE, @@ -1313,10 +1706,18 @@ def _negative_checks(run_id: str, source_ref: str) -> tuple[NegativeCheck, ...]: "42501", ), NegativeCheck( - "legacy_teleo_restore_read", + "legacy_database_connect", LEGACY_DATABASE, - _transactional("select 1 from teleo_restore.response_audit limit 1"), + "select 1 /* legacy_database_connect */;", "42501", + expected_connection_denial=True, + ), + NegativeCheck( + "template_database_connect", + TEMPLATE_DATABASE, + "select 1 /* template_database_connect */;", + "42501", + expected_connection_denial=True, ), ) @@ -1341,6 +1742,24 @@ def _expect_sqlstate( observed_sqlstate="success", ) + if check.expected_connection_denial: + normalized = result.stderr.casefold() + expected_database = f'permission denied for database "{check.database}"'.casefold() + if expected_database not in normalized or "does not have connect privilege" not in normalized: + raise VerificationError( + "negative_connection_denial_mismatch", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate="startup_error_unclassified", + ) + return { + "check": check.name, + "expected_sqlstate": check.expected_sqlstate, + "observed_error_class": "connect_privilege_denied", + "observed_sqlstate": "not_exposed_by_libpq_startup", + "result": "denied", + } + observed = SQLSTATE_RE.findall(result.stderr) unique_observed = sorted(set(observed)) if not observed: @@ -1373,6 +1792,7 @@ def _assert_identity(identity: dict[str, Any]) -> dict[str, Any]: "server_addr": PRIVATE_CLOUDSQL_HOST, "server_port": PRIVATE_CLOUDSQL_PORT, "ssl": True, + "system_identifier": EXPECTED_SYSTEM_IDENTIFIER, } if any(identity.get(key) != value for key, value in expected.items()): raise VerificationError("private_ssl_identity_mismatch", "database_identity") @@ -1510,16 +1930,6 @@ def _assert_catalog_privilege_posture(posture: dict[str, Any]) -> dict[str, int return sanitized -def _assert_legacy_catalog_privilege_posture(posture: dict[str, Any]) -> dict[str, int]: - sanitized: dict[str, int] = {} - for field in LEGACY_CATALOG_ZERO_COUNT_FIELDS: - value = posture.get(field) - if isinstance(value, bool) or not isinstance(value, int) or value != 0: - raise VerificationError("legacy_catalog_privilege_posture_mismatch", "legacy_catalog_privileges") - sanitized[field] = 0 - return sanitized - - def _assert_allowed_reads(readback: dict[str, Any]) -> dict[str, int]: sanitized: dict[str, int] = {} for key in ("claims_rows_sampled", "proposal_rows_sampled"): @@ -1553,6 +1963,7 @@ def verify_runtime_permissions( runner: Runner = subprocess_runner, base_env: Mapping[str, str] | None = None, effective_user: str | None = None, + dependency_validator: DependencyValidator = validate_runtime_dependencies, ) -> dict[str, Any]: try: run_id = validate_run_id(run_id) @@ -1566,14 +1977,13 @@ def verify_runtime_permissions( raise VerificationError("unix_user_unknown", "runtime_user") from None if effective_user != RUNTIME_UNIX_USER: raise VerificationError("unexpected_unix_user", "runtime_user") + dependency_validator() source = os.environ if base_env is None else base_env source_ref = f"leo-runtime-permission:{run_id}" generated_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") - with tempfile.TemporaryDirectory(prefix="leoclean-runtime-permission-gcloud-") as config_name: - config_dir = Path(config_name) - config_dir.chmod(0o700) + with nullcontext(selected_cloudsdk_config_path()) as config_dir: gcloud_env = _gcloud_environment(source, config_dir) password = _read_scoped_password(env=gcloud_env, runner=runner) psql_env = _psql_environment(source, password) @@ -1650,18 +2060,6 @@ def verify_runtime_permissions( "catalog_privilege_posture", ) ) - legacy_catalog_privileges = _assert_legacy_catalog_privilege_posture( - _parse_json_object( - _run_psql_success( - check="legacy_catalog_privileges", - database=LEGACY_DATABASE, - sql=_legacy_catalog_privilege_posture_sql(), - env=psql_env, - runner=runner, - ), - "legacy_catalog_privileges", - ) - ) allowed_reads = _assert_allowed_reads( _parse_json_object( _run_psql_success( @@ -1721,7 +2119,6 @@ def verify_runtime_permissions( "canary_rows_after": rows_after, "canary_rows_before": rows_before, "function_privileges": function_privileges, - "legacy_catalog_privileges": legacy_catalog_privileges, "negative_permissions": negative_permissions, "role_posture": role_posture, "stage_function_definition": stage_function_definition, @@ -1768,7 +2165,7 @@ def verify_runtime_permissions( "port": PRIVATE_CLOUDSQL_PORT, "project": PROJECT_ID, "role": RUNTIME_DATABASE_ROLE, - "sslmode": "require", + "sslmode": "verify-ca", }, } diff --git a/ops/verify_gcp_leoclean_service_environment.py b/ops/verify_gcp_leoclean_service_environment.py index 8dff5a1..d278d7f 100755 --- a/ops/verify_gcp_leoclean_service_environment.py +++ b/ops/verify_gcp_leoclean_service_environment.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 -I """Verify the effective Leo service environment without exposing its values.""" from __future__ import annotations @@ -18,6 +18,9 @@ RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime" SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password" RUNTIME_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +RUNTIME_CLOUDSDK_CONFIG = "/usr/local/libexec/livingip/leoclean-kb/gcloud-config" +RUNTIME_SSL_MODE = "verify-ca" +RUNTIME_SSL_ROOT_CERT = "/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem" FORBIDDEN_EXACT_FIELDS = frozenset( { @@ -26,6 +29,7 @@ FORBIDDEN_EXACT_FIELDS = frozenset( "BASHOPTS", "CDPATH", "CURL_CA_BUNDLE", + "CREDENTIALS_DIRECTORY", "ENV", "GLOBIGNORE", "GOOGLE_APPLICATION_CREDENTIALS", @@ -33,12 +37,22 @@ FORBIDDEN_EXACT_FIELDS = frozenset( "HTTP_PROXY", "IFS", "NO_PROXY", + "OPENSSL_CONF", + "OPENSSL_ENGINES", + "OPENSSL_MODULES", "PS4", "REQUESTS_CA_BUNDLE", "SHELLOPTS", "SSL_CERT_DIR", "SSL_CERT_FILE", "SSLKEYLOGFILE", + "GCONV_PATH", + "TELEO_CLOUDSQL_CREDENTIAL_MODE", + "TELEO_KB_CLAIM_BASE_URL", + "TELEO_KB_READ_ATTEMPTS", + "TELEO_MEMORY_INCLUDE_EXCERPTS", + "TELEO_MEMORY_REDACT", + "XDG_CONFIG_HOME", } ) @@ -83,7 +97,12 @@ def expected_environment() -> dict[str, str]: """Build the exact reviewed environment contract.""" return { + "CLOUDSDK_CONFIG": RUNTIME_CLOUDSDK_CONFIG, "PATH": RUNTIME_PATH, + "PGSSLMODE": RUNTIME_SSL_MODE, + "PGSSLROOTCERT": RUNTIME_SSL_ROOT_CERT, + "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", "TELEO_CANONICAL_CLOUDSQL_DB": CANONICAL_DATABASE, "TELEO_CLOUDSQL_DB": CANONICAL_DATABASE, "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK": "0", @@ -127,22 +146,31 @@ def _forbidden_field_labels(environment: Mapping[str, str]) -> tuple[str, ...]: labels: set[str] = set() for key in environment: normalized = key.upper() - if normalized.startswith("PG"): + if normalized.startswith("PG") and normalized not in {"PGSSLMODE", "PGSSLROOTCERT"}: labels.add("PG*") - if normalized.startswith("CLOUDSDK_"): + if normalized.startswith("CLOUDSDK_") and normalized != "CLOUDSDK_CONFIG": labels.add("CLOUDSDK_*") if normalized.startswith("LD_"): labels.add("LD_*") if normalized.startswith("BASH_FUNC_"): labels.add("BASH_FUNC_*") - if normalized.startswith("PYTHON") and normalized != "PYTHONUNBUFFERED": + if normalized.startswith("PYTHON") and normalized not in { + "PYTHONNOUSERSITE", + "PYTHONSAFEPATH", + "PYTHONUNBUFFERED", + }: labels.add("PYTHON*") if normalized in FORBIDDEN_EXACT_FIELDS: labels.add(normalized) return tuple(sorted(labels)) -def validate_environment(environment: Mapping[str, str]) -> tuple[str, ...]: +def validate_environment( + environment: Mapping[str, str], + *, + allow_missing_cloudsdk_config: bool = False, + allow_missing_runtime_security_fields: bool = False, +) -> tuple[str, ...]: """Validate the effective environment and return only reviewed field names.""" if not isinstance(environment, Mapping) or any( @@ -151,6 +179,18 @@ def validate_environment(environment: Mapping[str, str]) -> tuple[str, ...]: raise VerificationError("environment_malformed") expected = expected_environment() + if allow_missing_runtime_security_fields: + for field in ( + "CLOUDSDK_CONFIG", + "PGSSLMODE", + "PGSSLROOTCERT", + "PYTHONNOUSERSITE", + "PYTHONSAFEPATH", + ): + if field not in environment: + expected.pop(field) + if allow_missing_cloudsdk_config and "CLOUDSDK_CONFIG" not in environment: + expected.pop("CLOUDSDK_CONFIG") administrator_secret = ADMINISTRATOR_PASSWORD_SECRET.casefold() if any(administrator_secret in value.casefold() for value in environment.values()): raise VerificationError("administrator_secret_reference") @@ -172,12 +212,18 @@ def verify_process_environment( pid: int, *, proc_root: Path = Path("/proc"), + allow_missing_cloudsdk_config: bool = False, + allow_missing_runtime_security_fields: bool = False, ) -> dict[str, object]: """Read and verify one process environment, returning a sanitized receipt.""" validated_pid = validate_pid(pid) raw = (proc_root / str(validated_pid) / "environ").read_bytes() - validated_fields = validate_environment(parse_environment(raw)) + validated_fields = validate_environment( + parse_environment(raw), + allow_missing_cloudsdk_config=allow_missing_cloudsdk_config, + allow_missing_runtime_security_fields=allow_missing_runtime_security_fields, + ) return { "runtime_environment": "scoped", "status": "pass", @@ -196,13 +242,19 @@ def canonical_json(payload: Mapping[str, Any]) -> str: def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = _SanitizedArgumentParser(description=__doc__, add_help=False) parser.add_argument("--pid", required=True) + parser.add_argument("--allow-missing-cloudsdk-config", action="store_true") + parser.add_argument("--allow-missing-runtime-security-fields", action="store_true") return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: try: args = parse_args(argv) - receipt = verify_process_environment(validate_pid(args.pid)) + receipt = verify_process_environment( + validate_pid(args.pid), + allow_missing_cloudsdk_config=args.allow_missing_cloudsdk_config, + allow_missing_runtime_security_fields=args.allow_missing_runtime_security_fields, + ) returncode = 0 except VerificationError as exc: receipt = failure_receipt(exc) diff --git a/scripts/run_gcp_generated_db_direct_claim_suite.py b/scripts/run_gcp_generated_db_direct_claim_suite.py index f335731..6f2cc0a 100644 --- a/scripts/run_gcp_generated_db_direct_claim_suite.py +++ b/scripts/run_gcp_generated_db_direct_claim_suite.py @@ -46,8 +46,10 @@ DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service" DEFAULT_HOST = "10.61.0.3" DEFAULT_PROJECT = "teleo-501523" DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" +DEFAULT_SSL_ROOT_CERT = Path("/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem") +DEFAULT_SYSTEM_IDENTIFIER = "7659718422914359312" DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql" -REVIEWED_CLOUDSQL_TOOL_SHA256 = "66fa84b3bd6fe67e102fbce4509ea28c3537ba5bb1091e1a1a457b7425366064" +REVIEWED_CLOUDSQL_TOOL_SHA256 = "17d5213756ea6492e4232784a04b7f584afee5dce5184b33a2227e58847472c1" REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a" COUNT_READBACK_RE = re.compile( r"DB readback:\s*claims:\s*`?(?P\d+)`?;\s*" @@ -192,10 +194,13 @@ def run_target_psql( **os.environ, "PGPASSWORD": password, "PGOPTIONS": "-c default_transaction_read_only=on", + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": str(args.ssl_root_cert), } command = [ "psql", - f"host={args.host} port=5432 dbname={args.target_db} user=postgres sslmode=require connect_timeout=8", + f"host={args.host} port=5432 dbname={args.target_db} user=postgres " + f"sslmode=verify-ca sslrootcert={args.ssl_root_cert} connect_timeout=8", "-X", "-Atq", "-v", @@ -232,7 +237,11 @@ rollback; return json.loads(rows[-1]) -def validate_database_identity(identity: dict[str, Any], target_db: str) -> None: +def validate_database_identity( + identity: dict[str, Any], + target_db: str, + expected_system_identifier: str = DEFAULT_SYSTEM_IDENTIFIER, +) -> None: if identity.get("current_database") != target_db: raise bound.CheckpointError("Cloud SQL identity does not match the generated target database") if identity.get("ssl") is not True: @@ -252,8 +261,8 @@ def validate_database_identity(identity: dict[str, Any], target_db: str) -> None ) if not any(address in network for network in private_networks): raise bound.CheckpointError("Cloud SQL target address is not RFC1918-private") - if not identity.get("system_identifier"): - raise bound.CheckpointError("Cloud SQL target system identifier is missing") + if identity.get("system_identifier") != expected_system_identifier: + raise bound.CheckpointError("Cloud SQL target system identifier is not the reviewed instance") def canonical_status(args: argparse.Namespace) -> dict[str, Any]: @@ -347,6 +356,7 @@ def build_cloudsql_wrapper( host: str, project: str, password_secret: str, + ssl_root_cert: Path, run_nonce: str, system_exec_path: str = bound.SYSTEM_EXEC_PATH, ) -> str: @@ -361,18 +371,21 @@ TARGET_HOST={shlex.quote(host)} SOURCE_COMPUTE={shlex.quote(SOURCE_COMPUTE)} PROJECT={shlex.quote(project)} PASSWORD_SECRET={shlex.quote(password_secret)} +SSL_ROOT_CERT={shlex.quote(str(ssl_root_cert))} RUN_NONCE={shlex.quote(run_nonce)} PYTHON={shlex.quote(python)} export PATH={shlex.quote(system_exec_path)} export CLOUDSDK_CONFIG=/home/teleo/.config/gcloud export PGOPTIONS="-c default_transaction_read_only=on" +export PGSSLMODE=verify-ca +export PGSSLROOTCERT="$SSL_ROOT_CERT" INVOCATION_ID="$($PYTHON -c 'import uuid; print(uuid.uuid4())')" if ! PGPASSWORD="$(gcloud secrets versions access latest --secret="$PASSWORD_SECRET" --project="$PROJECT")"; then exit 124 fi export PGPASSWORD trap 'unset PGPASSWORD PGOPTIONS' EXIT -DB_RECEIPT="$(psql "host=$TARGET_HOST port=5432 dbname=$TARGET_DATABASE user=postgres sslmode=require connect_timeout=8" -X -Atq -v ON_ERROR_STOP=1 <<'SQL' +DB_RECEIPT="$(psql "host=$TARGET_HOST port=5432 dbname=$TARGET_DATABASE user=postgres sslmode=verify-ca sslrootcert=$SSL_ROOT_CERT connect_timeout=8" -X -Atq -v ON_ERROR_STOP=1 <<'SQL' begin transaction read only; select jsonb_build_object( 'current_database', current_database(), @@ -468,6 +481,7 @@ def patch_temp_bridge(args: argparse.Namespace, temp_profile: Path) -> dict[str, project=args.project, password_secret=args.password_secret, run_nonce=run_nonce, + ssl_root_cert=args.ssl_root_cert, ) binding = f""" ## Generated GCP Database Checkpoint @@ -788,6 +802,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--host", default=DEFAULT_HOST) parser.add_argument("--project", default=DEFAULT_PROJECT) parser.add_argument("--password-secret", default=DEFAULT_SECRET) + parser.add_argument("--ssl-root-cert", default=DEFAULT_SSL_ROOT_CERT, type=Path) parser.add_argument("--service", default=DEFAULT_SERVICE) parser.add_argument("--live-profile", default=bound.LIVE_PROFILE, type=Path) parser.add_argument("--chat-id", default=bound.DEFAULT_CHAT_ID) @@ -804,6 +819,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error("--target-db must start with teleo_clone_ and cannot name a live database") if not args.cloudsql_tool.is_file(): parser.error("--cloudsql-tool must exist") + if not args.ssl_root_cert.is_file(): + parser.error("--ssl-root-cert must exist") if not args.manifest_sql.is_file(): parser.error("--manifest-sql must exist") if not args.parity_receipt.is_file(): diff --git a/scripts/run_gcp_generated_db_working_leo_suite.py b/scripts/run_gcp_generated_db_working_leo_suite.py index 81eb247..d3bc26e 100644 --- a/scripts/run_gcp_generated_db_working_leo_suite.py +++ b/scripts/run_gcp_generated_db_working_leo_suite.py @@ -452,7 +452,8 @@ select current_database() = :'expected_database' as exact_database_bound \\gset "psql", ( f"host={self.args.host} port=5432 dbname={self.args.target_db} " - f"user={role} sslmode=require connect_timeout=8" + f"user={role} sslmode=verify-ca " + f"sslrootcert={self.args.ssl_root_cert} connect_timeout=8" ), "-X", "-Atq", @@ -461,7 +462,13 @@ select current_database() = :'expected_database' as exact_database_bound \\gset "-v", f"expected_database={self.args.target_db}", ] - env = {**os.environ, "PGPASSWORD": password, "PGOPTIONS": pgoptions} + env = { + **os.environ, + "PGPASSWORD": password, + "PGOPTIONS": pgoptions, + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": str(self.args.ssl_root_cert), + } try: try: output = gcp.run( @@ -474,6 +481,8 @@ select current_database() = :'expected_database' as exact_database_bound \\gset finally: env.pop("PGPASSWORD", None) env.pop("PGOPTIONS", None) + env.pop("PGSSLMODE", None) + env.pop("PGSSLROOTCERT", None) password = "" lines = output.splitlines() receipt_lines = [line for line in lines if line.startswith(ROLE_RECEIPT_PREFIX)] @@ -692,6 +701,7 @@ def _build_gcp_read_wrapper( host=args.host, project=args.project, password_secret=args.password_secret, + ssl_root_cert=getattr(args, "ssl_root_cert", gcp.DEFAULT_SSL_ROOT_CERT), run_nonce=run_nonce, ) wrapper = wrapper.replace( @@ -699,7 +709,11 @@ def _build_gcp_read_wrapper( f"PASSWORD_SECRET={shlex.quote(args.password_secret)}\nREAD_ROLE={shlex.quote(args.read_role)}\n", 1, ) - wrapper = wrapper.replace("user=postgres sslmode=require", "user=$READ_ROLE sslmode=require", 1) + wrapper = wrapper.replace( + "user=postgres sslmode=verify-ca", + "user=$READ_ROLE sslmode=verify-ca", + 1, + ) wrapper = wrapper.replace( " 'default_transaction_read_only', current_setting('default_transaction_read_only')\n", " 'default_transaction_read_only', current_setting('default_transaction_read_only'),\n" @@ -755,6 +769,8 @@ def _cloudsql_lifecycle_wrapper( args.instance, "--password-secret", args.password_secret, + "--ssl-root-cert", + str(args.ssl_root_cert), "--read-role", args.read_role, "--stage-password-secret", @@ -1600,6 +1616,7 @@ def parse_internal_stage_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--project", required=True) parser.add_argument("--instance", required=True) parser.add_argument("--password-secret", required=True) + parser.add_argument("--ssl-root-cert", required=True, type=Path) parser.add_argument("--read-role", required=True) parser.add_argument("--stage-password-secret", required=True) parser.add_argument("--stage-role", required=True) @@ -1609,6 +1626,8 @@ def parse_internal_stage_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--target-identity-receipt", required=True, type=Path) parser.add_argument("--target-identity-sha256", required=True) args = parser.parse_args(argv) + if not args.ssl_root_cert.is_file(): + parser.error("--ssl-root-cert must exist") args.review_role = "kb_review" args.apply_role = "kb_apply" args.allow_stage = True @@ -1680,6 +1699,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--project", default=gcp.DEFAULT_PROJECT) parser.add_argument("--instance") parser.add_argument("--password-secret", default=gcp.DEFAULT_SECRET) + parser.add_argument("--ssl-root-cert", default=gcp.DEFAULT_SSL_ROOT_CERT, type=Path) parser.add_argument("--read-role", default="kb_read") parser.add_argument("--stage-role", default=DEFAULT_STAGE_ROLE) parser.add_argument("--review-role", default=DEFAULT_REVIEW_ROLE) @@ -1713,6 +1733,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: if args.turn_timeout <= 0: parser.error("--turn-timeout must be positive") if args.execute: + if not args.ssl_root_cert.is_file(): + parser.error("--ssl-root-cert must exist for execution") receipt_inputs = ( args.target_identity_receipt, args.target_identity_sha256, diff --git a/systemd/leoclean-gcp-prod-parallel-cloudsql.conf b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf index 68d4d34..e8768fc 100644 --- a/systemd/leoclean-gcp-prod-parallel-cloudsql.conf +++ b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf @@ -1,17 +1,36 @@ [Service] UnsetEnvironment=PGPASSWORD UnsetEnvironment=PGPASSFILE PGSERVICE PGHOST PGPORT PGDATABASE PGUSER PGOPTIONS -UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN +UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS CDPATH GLOBIGNORE IFS PS4 UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT LD_DEBUG LD_DEBUG_OUTPUT LD_DYNAMIC_WEAK LD_HWCAP_MASK LD_ORIGIN_PATH LD_PROFILE LD_SHOW_AUXV LD_TRACE_LOADED_OBJECTS LD_USE_LOAD_BIAS LD_VERBOSE LD_WARN UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT PYTHONUSERBASE PYTHONHTTPSVERIFY UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE -ReadOnlyPaths=/home/teleo/.hermes +UnsetEnvironment=OPENSSL_CONF OPENSSL_MODULES OPENSSL_ENGINES GCONV_PATH CREDENTIALS_DIRECTORY +UnsetEnvironment=XDG_CONFIG_HOME +UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT +ReadOnlyPaths= +ReadWritePaths= +InaccessiblePaths= +BindPaths= +BindReadOnlyPaths= +SupplementaryGroups= +ReadOnlyPaths=/home/teleo ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state /home/teleo/.hermes/profiles/leoclean/workspace +InaccessiblePaths=-/home/teleo/.config/gcloud -/home/teleo/.pgpass -/home/teleo/.pg_service.conf -/home/teleo/.postgresql BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin -CapabilityBoundingSet=~CAP_SYS_ADMIN +CapabilityBoundingSet= +AmbientCapabilities= +NoNewPrivileges=yes +Group=teleo +SupplementaryGroups=teleo Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +Environment=CLOUDSDK_CONFIG=/usr/local/libexec/livingip/leoclean-kb/gcloud-config +Environment=PGSSLMODE=verify-ca +Environment=PGSSLROOTCERT=/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem +Environment=PYTHONNOUSERSITE=1 +Environment=PYTHONSAFEPATH=1 Environment=TELEO_KB_MODE=cloudsql Environment=TELEO_GCP_METADATA_ONLY=1 Environment=TELEO_GCP_PROJECT=teleo-501523 diff --git a/tests/test_gcp_generated_db_direct_claim_suite.py b/tests/test_gcp_generated_db_direct_claim_suite.py index 475b8e7..08c72d3 100644 --- a/tests/test_gcp_generated_db_direct_claim_suite.py +++ b/tests/test_gcp_generated_db_direct_claim_suite.py @@ -109,6 +109,7 @@ def test_wrapper_binds_private_read_only_database_and_records_calls(tmp_path: Pa host="10.61.0.3", project="teleo-501523", password_secret="test-secret-name", + ssl_root_cert=tmp_path / "cloudsql-server-ca.pem", run_nonce="nonce", ) @@ -117,7 +118,8 @@ def test_wrapper_binds_private_read_only_database_and_records_calls(tmp_path: Pa assert '--canonical-db "$TARGET_DATABASE"' in wrapper assert "--credential-mode clone-readonly" in wrapper assert "--user postgres" in wrapper - assert "sslmode=require" in wrapper + assert "sslmode=verify-ca" in wrapper + assert "sslrootcert=$SSL_ROOT_CERT" in wrapper assert "begin transaction read only" in wrapper assert 'export PGOPTIONS="-c default_transaction_read_only=on"' in wrapper assert "default_transaction_read_only" in wrapper @@ -140,6 +142,7 @@ def test_database_identity_requires_private_tls_and_read_only_target() -> None: "default_transaction_read_only": "on", }, "teleo_clone_test", + "1234", ) with pytest.raises(RuntimeError, match="not RFC1918-private"): @@ -153,6 +156,7 @@ def test_database_identity_requires_private_tls_and_read_only_target() -> None: "default_transaction_read_only": "on", }, "teleo_clone_test", + "1234", ) @@ -184,6 +188,7 @@ def test_database_fingerprint_uses_full_normalized_manifest(monkeypatch: pytest. "host": "10.61.0.3", "target_db": "teleo_clone_test", "manifest_sql": manifest_sql, + "ssl_root_cert": tmp_path / "cloudsql-server-ca.pem", }, )() output = "\n".join( @@ -282,6 +287,7 @@ def test_generated_wrapper_executes_only_clone_bound_default_read_only_tool(tmp_ host="10.61.0.3", project="teleo-501523", password_secret="test-secret", + ssl_root_cert=tmp_path / "cloudsql-server-ca.pem", run_nonce="test-nonce", system_exec_path=f"{fake_bin}:/usr/bin:/bin", ) diff --git a/tests/test_gcp_generated_db_working_leo_suite.py b/tests/test_gcp_generated_db_working_leo_suite.py index a3e5526..4ac006c 100644 --- a/tests/test_gcp_generated_db_working_leo_suite.py +++ b/tests/test_gcp_generated_db_working_leo_suite.py @@ -131,6 +131,7 @@ def executor_args(**overrides: object) -> argparse.Namespace: "project": "teleo-501523", "instance": "teleo-pgvector-standby", "password_secret": "read-secret", + "ssl_root_cert": Path("ops/gcp-teleo-pgvector-standby-server-ca.pem").resolve(), "read_role": "kb_read", "stage_role": "kb_stage_writer", "review_role": "kb_review", @@ -229,6 +230,10 @@ def test_executor_binds_every_call_and_defaults_read_only_except_controller_phas assert len(calls) == 4 assert all("dbname=teleo_clone_working_leo" in call["command"][1] for call in calls) + assert all("sslmode=verify-ca" in call["command"][1] for call in calls) + assert all("sslrootcert=" in call["command"][1] for call in calls) + assert all(call["env"]["PGSSLMODE"] == "verify-ca" for call in calls) + assert all(call["env"]["PGSSLROOTCERT"].endswith("gcp-teleo-pgvector-standby-server-ca.pem") for call in calls) assert all("expected_database=teleo_clone_working_leo" in call["command"] for call in calls) assert calls[0]["env"]["PGOPTIONS"] == "-c default_transaction_read_only=on" assert [call["env"]["PGOPTIONS"] for call in calls[1:]] == [ @@ -535,6 +540,9 @@ def test_lifecycle_wrapper_exposes_only_read_or_exact_stage_and_never_delivery(t ) assert catalog_wrapper.count('"prompt_id": "DC-01"') == 2 assert '--user "$READ_ROLE"' in catalog_wrapper + assert "sslmode=verify-ca" in catalog_wrapper + assert "sslrootcert=$SSL_ROOT_CERT" in catalog_wrapper + assert "sslmode=require" not in catalog_wrapper def test_capability_receipt_rejects_superuser_or_wrong_phase_acl() -> None: diff --git a/tests/test_gcp_leoclean_runtime_permissions.py b/tests/test_gcp_leoclean_runtime_permissions.py index ca97995..8f594ff 100644 --- a/tests/test_gcp_leoclean_runtime_permissions.py +++ b/tests/test_gcp_leoclean_runtime_permissions.py @@ -31,7 +31,6 @@ class FakeRunner: function_posture_override: Mapping[str, Mapping[str, object]] | None = None, stage_definition_override: Mapping[str, object] | None = None, catalog_posture_override: Mapping[str, object] | None = None, - legacy_catalog_posture_override: Mapping[str, object] | None = None, ) -> None: self.sqlstate_override = sqlstate_override self.sqlstate_missing_for = sqlstate_missing_for @@ -46,7 +45,6 @@ class FakeRunner: } self.stage_definition_override = dict(stage_definition_override or {}) self.catalog_posture_override = dict(catalog_posture_override or {}) - self.legacy_catalog_posture_override = dict(legacy_catalog_posture_override or {}) self.calls: list[tuple[list[str], dict[str, str], int, bool]] = [] self.negative = {check.sql: check for check in verifier._negative_checks(RUN_ID, SOURCE_REF)} @@ -94,6 +92,7 @@ class FakeRunner: "session_user": verifier.RUNTIME_DATABASE_ROLE, "ssl": True, "ssl_version": "TLSv1.3", + "system_identifier": verifier.EXPECTED_SYSTEM_IDENTIFIER, } identity.update(self.identity_override) return verifier.CommandResult(0, json.dumps(identity), f"ignored {SECRET_VALUE}") @@ -179,13 +178,6 @@ class FakeRunner: ) catalog_posture.update(self.catalog_posture_override) return verifier.CommandResult(0, json.dumps(catalog_posture), f"ignored {SECRET_VALUE}") - if sql == verifier._legacy_catalog_privilege_posture_sql(): - legacy_posture: dict[str, object] = { - field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS - } - legacy_posture["leak"] = SECRET_VALUE - legacy_posture.update(self.legacy_catalog_posture_override) - return verifier.CommandResult(0, json.dumps(legacy_posture), f"ignored {SECRET_VALUE}") if sql == verifier._canary_count_sql(SOURCE_REF): return verifier.CommandResult(0, "0\n", f"ignored {SECRET_VALUE}") if sql == verifier._stage_sql(RUN_ID, SOURCE_REF): @@ -211,6 +203,15 @@ class FakeRunner: return verifier.CommandResult(0, SECRET_VALUE, f"unexpected success {SECRET_VALUE}") if check.name == self.sqlstate_missing_for: return verifier.CommandResult(1, SECRET_VALUE, f"denied without state {SECRET_VALUE}") + if check.expected_connection_denial: + return verifier.CommandResult( + 2, + SECRET_VALUE, + ( + f'connection failed: FATAL: permission denied for database "{check.database}"\n' + f"DETAIL: User does not have CONNECT privilege. {SECRET_VALUE}" + ), + ) observed = check.expected_sqlstate if self.sqlstate_override is not None and check.name == self.sqlstate_override[0]: observed = self.sqlstate_override[1] @@ -243,6 +244,7 @@ def run_success(runner: FakeRunner) -> dict[str, object]: runner=runner, base_env=poisoned_environment(), effective_user="teleo", + dependency_validator=lambda: None, ) @@ -253,8 +255,10 @@ def assert_child_boundaries(runner: FakeRunner) -> None: assert timeout == verifier.COMMAND_TIMEOUT_SECONDS pg_keys = sorted(key for key in env if key.upper().startswith("PG")) if command[0] == verifier.PSQL_BIN: - assert pg_keys == ["PGPASSWORD"] + assert pg_keys == ["PGPASSWORD", "PGSSLMODE", "PGSSLROOTCERT"] assert env["PGPASSWORD"] == SECRET_VALUE + assert env["PGSSLMODE"] == "verify-ca" + assert env["PGSSLROOTCERT"] == str(verifier.SERVER_CA_PATH) assert discard_stdout is False else: assert pg_keys == [] @@ -284,6 +288,7 @@ def test_live_receipt_contract_is_sanitized_and_uses_exact_child_environments() "session_user": verifier.RUNTIME_DATABASE_ROLE, "ssl": True, "ssl_version": "TLSv1.3", + "system_identifier": verifier.EXPECTED_SYSTEM_IDENTIFIER, } assert receipt["checks"]["rolled_back_proposal"] == { "agent_id_matches": True, @@ -331,9 +336,6 @@ def test_live_receipt_contract_is_sanitized_and_uses_exact_child_environments() **{field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS}, **{field: True for field in verifier.CATALOG_TRUE_FIELDS}, } - assert receipt["checks"]["legacy_catalog_privileges"] == { - field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS - } assert receipt["checks"]["stage_function_definition"] == { "acl_exact": True, "argument_modes": None, @@ -570,7 +572,6 @@ def test_catalog_queries_use_exact_regprocedure_signatures_and_both_membership_d owner_role_sql = verifier._stage_owner_role_posture_sql() stage_definition_sql = verifier._stage_function_definition_sql() catalog_sql = verifier._catalog_privilege_posture_sql() - legacy_sql = verifier._legacy_catalog_privilege_posture_sql() assert "pg_catalog.to_regprocedure(signature)" in function_sql for _name, signature, _exists, _execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS: @@ -597,12 +598,10 @@ def test_catalog_queries_use_exact_regprocedure_signatures_and_both_membership_d assert "nspname !~ '^pg_'" in catalog_sql assert "nspname <> 'information_schema'" in catalog_sql assert verifier.STAGE_OWNER_DATABASE_ROLE in catalog_sql - assert "teleo_restore" in legacy_sql - assert "pg_catalog.has_schema_privilege" in legacy_sql - assert "pg_catalog.has_table_privilege" in legacy_sql - assert "pg_catalog.has_column_privilege" in legacy_sql - assert "pg_catalog.has_sequence_privilege" in legacy_sql - assert "pg_catalog.has_function_privilege" in legacy_sql + assert "public_database_connect_grants" in catalog_sql + assert "public_large_object_mutation_routine_execute" in catalog_sql + assert "'lo_creat'" in catalog_sql + assert "'lo_open'" in catalog_sql for schema, relation in verifier.READ_TABLE_ALLOWLIST: assert schema in catalog_sql assert relation in catalog_sql @@ -630,38 +629,27 @@ def test_any_missing_required_catalog_privilege_fails_closed(field: str) -> None assert caught.value.code == "catalog_privilege_posture_mismatch" -@pytest.mark.parametrize("field", verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS) -def test_any_teleo_restore_catalog_privilege_fails_closed(field: str) -> None: - runner = FakeRunner(legacy_catalog_posture_override={field: 1}) - - with pytest.raises(verifier.VerificationError) as caught: - run_success(runner) - - assert caught.value.code == "legacy_catalog_privilege_posture_mismatch" - assert caught.value.check == "legacy_catalog_privileges" - assert SECRET_VALUE not in str(caught.value) - - -def test_teleo_restore_catalog_and_behavioral_denial_are_both_exercised_on_legacy_database() -> None: +def test_legacy_and_template_database_connect_are_both_behaviorally_denied() -> None: runner = FakeRunner() receipt = run_success(runner) - legacy_catalog_call = next( - command + denials = {check["check"]: check for check in receipt["checks"]["negative_permissions"]} + for name in ("legacy_database_connect", "template_database_connect"): + assert denials[name] == { + "check": name, + "expected_sqlstate": "42501", + "observed_error_class": "connect_privilege_denied", + "observed_sqlstate": "not_exposed_by_libpq_startup", + "result": "denied", + } + database_calls = [ + " ".join(command) for command, _env, _timeout, _discard in runner.calls - if any(part == f"--command={verifier._legacy_catalog_privilege_posture_sql()}" for part in command) - ) - assert any(f"dbname={verifier.LEGACY_DATABASE}" in part for part in legacy_catalog_call) - legacy_denial = next( - check for check in receipt["checks"]["negative_permissions"] if check["check"] == "legacy_teleo_restore_read" - ) - assert legacy_denial == { - "check": "legacy_teleo_restore_read", - "expected_sqlstate": "42501", - "observed_sqlstate": "42501", - "result": "denied", - } + if command[0] == verifier.PSQL_BIN + ] + assert any(f"dbname={verifier.LEGACY_DATABASE}" in command for command in database_calls) + assert any(f"dbname={verifier.TEMPLATE_DATABASE}" in command for command in database_calls) def test_required_sqlstate_mismatch_fails_closed_without_secret_in_error() -> None: @@ -693,6 +681,19 @@ def test_missing_verbose_sqlstate_fails_closed_without_raw_stderr() -> None: assert SECRET_VALUE not in json.dumps(error.as_dict()) +def test_unclassified_database_startup_failure_cannot_masquerade_as_connect_denial() -> None: + runner = FakeRunner(sqlstate_missing_for="legacy_database_connect") + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + error = caught.value + assert error.code == "negative_connection_denial_mismatch" + assert error.check == "legacy_database_connect" + assert error.observed_sqlstate == "startup_error_unclassified" + assert SECRET_VALUE not in json.dumps(error.as_dict()) + + def test_unexpected_permission_success_fails_closed_and_transaction_sql_has_rollback() -> None: runner = FakeRunner(unexpected_success_for="proposal_update") diff --git a/tests/test_gcp_leoclean_service_environment.py b/tests/test_gcp_leoclean_service_environment.py index a1be9e6..449ed8d 100644 --- a/tests/test_gcp_leoclean_service_environment.py +++ b/tests/test_gcp_leoclean_service_environment.py @@ -78,7 +78,6 @@ def test_later_dropin_effective_override_fails_without_outputting_value( [ ("PGPASSWORD", "PG*"), ("pgservice", "PG*"), - ("CLOUDSDK_CONFIG", "CLOUDSDK_*"), ("CLOUDSDK_CORE_PROJECT", "CLOUDSDK_*"), ("CLOUDSDK_AUTH_ACCESS_TOKEN", "CLOUDSDK_*"), ("cloudsdk_auth_refresh_token", "CLOUDSDK_*"), @@ -102,6 +101,11 @@ def test_later_dropin_effective_override_fails_without_outputting_value( ("REQUESTS_CA_BUNDLE", "REQUESTS_CA_BUNDLE"), ("curl_ca_bundle", "CURL_CA_BUNDLE"), ("SSLKEYLOGFILE", "SSLKEYLOGFILE"), + ("TELEO_CLOUDSQL_CREDENTIAL_MODE", "TELEO_CLOUDSQL_CREDENTIAL_MODE"), + ("TELEO_KB_CLAIM_BASE_URL", "TELEO_KB_CLAIM_BASE_URL"), + ("TELEO_KB_READ_ATTEMPTS", "TELEO_KB_READ_ATTEMPTS"), + ("TELEO_MEMORY_INCLUDE_EXCERPTS", "TELEO_MEMORY_INCLUDE_EXCERPTS"), + ("TELEO_MEMORY_REDACT", "TELEO_MEMORY_REDACT"), ], ) def test_environment_file_broader_credentials_and_forbidden_fields_fail_closed( diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index 3a41541..90105f1 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -26,6 +26,8 @@ ROOT = Path(__file__).resolve().parents[1] BRIDGE_DIR = ROOT / "hermes-agent" / "leoclean-bin" DB_CONTEXT_PLUGIN = ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py" GCP_RUNTIME_ROLE_SQL = ROOT / "ops" / "gcp_leoclean_runtime_role.sql" +GCP_RUNTIME_WRAPPER = BRIDGE_DIR / "teleo-kb-gcp" +GCP_RUNTIME_PROVISIONER = ROOT / "ops" / "provision_gcp_leoclean_runtime_role.sh" def test_leoclean_bridge_files_are_present_and_parseable() -> None: @@ -42,6 +44,71 @@ def test_leoclean_bridge_files_are_present_and_parseable() -> None: subprocess.run(["bash", "-n", str(wrapper)], check=True) +@pytest.mark.parametrize( + "argument", + [ + "--host", + "--host=127.0.0.1", + "--port", + "--port=6543", + "--db", + "--db=postgres", + "--canonical-db", + "--canonical-db=postgres", + "--user", + "--user=postgres", + "--password-secret", + "--password-secret=administrator-secret", + "--project", + "--project=other-project", + "--credential-mode", + "--credential-mode=clone-readonly", + ], +) +def test_deployed_gcp_wrapper_executable_rejects_every_identity_override(argument: str) -> None: + injected = "sensitive-override-value" + completed = subprocess.run( + [str(GCP_RUNTIME_WRAPPER), argument, injected], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 64 + assert "identity and endpoint overrides are forbidden" in completed.stderr + assert injected not in completed.stderr + assert completed.stdout == "" + + +def test_gcp_wrapper_and_password_provisioner_are_executable_and_syntax_valid() -> None: + assert os.access(GCP_RUNTIME_WRAPPER, os.X_OK) + assert os.access(GCP_RUNTIME_PROVISIONER, os.X_OK) + subprocess.run(["bash", "-n", str(GCP_RUNTIME_WRAPPER)], check=True) + subprocess.run(["bash", "-n", str(GCP_RUNTIME_PROVISIONER)], check=True) + + wrapper = GCP_RUNTIME_WRAPPER.read_text(encoding="utf-8") + assert "exec /usr/bin/env -i" in wrapper + assert '/usr/bin/python3 -I "$CLOUDSQL_TOOL" "$@"' in wrapper + assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in wrapper + assert "PGSSLMODE=verify-ca" in wrapper + assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in wrapper + + provisioner = GCP_RUNTIME_PROVISIONER.read_text(encoding="utf-8") + capture_runtime = "runtime_password=${TELEO_LEOCLEAN_DB_PASSWORD-}" + capture_admin = "administrator_password=${PGPASSWORD-}" + clear_ambient = "unset TELEO_LEOCLEAN_DB_PASSWORD PGPASSWORD" + first_external_child = 'SCRIPT_DIR="$(cd -- "$script_directory" && pwd -P)"' + assert provisioner.index(capture_runtime) < provisioner.index(first_external_child) + assert provisioner.index(capture_admin) < provisioner.index(first_external_child) + assert provisioner.index(clear_ambient) < provisioner.index(first_external_child) + assert '"$SETSID_BIN" -- "$PSQL_BIN"' in provisioner + assert 'kill -TERM -- "-$provision_pid"' in provisioner + assert 'wait "$provision_pid"' in provisioner + assert "SERVER_CA_SHA256=80701e" in provisioner + assert "--no-password" in provisioner + assert "PGSSLMODE=verify-ca" in provisioner + + def test_cloudsql_wrapper_supports_expected_review_gated_commands() -> None: wrapper_text = (BRIDGE_DIR / "teleo-kb").read_text() cloudsql_text = (BRIDGE_DIR / "cloudsql_memory_tool.py").read_text() @@ -502,7 +569,7 @@ def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pyt monkeypatch.setattr(module.subprocess, "run", fake_run) assert module.run_psql(args, "select 1;") == "ok\n" - assert captured["command"] == ["psql", "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"] + assert captured["command"] == ["/usr/bin/psql", "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"] kwargs = captured["kwargs"] assert isinstance(kwargs, dict) env = kwargs["env"] @@ -512,7 +579,8 @@ def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pyt "PGPORT": "5432", "PGDATABASE": "teleo_canonical", "PGUSER": module.RUNTIME_DB_USER, - "PGSSLMODE": "require", + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": module.RUNTIME_SSL_ROOT_CERT, "PGCONNECT_TIMEOUT": "8", "PGPASSWORD": "scoped-runtime-password", } @@ -728,7 +796,7 @@ def test_cloudsql_runtime_read_marker_binds_database_user_private_server_and_ssl "server_port": int(module.RUNTIME_CLOUDSQL_PORT), "ssl": True, "ssl_version": "TLSv1.3", - "system_identifier": "system-id", + "system_identifier": module.RUNTIME_SYSTEM_IDENTIFIER, "wal_lsn": "0/1", } captured: dict[str, object] = {} @@ -771,7 +839,7 @@ def test_cloudsql_runtime_read_marker_rejects_wrong_live_identity( "server_port": int(module.RUNTIME_CLOUDSQL_PORT), "ssl": True, "ssl_version": "TLSv1.3", - "system_identifier": "system-id", + "system_identifier": module.RUNTIME_SYSTEM_IDENTIFIER, "wal_lsn": "0/1", } marker[field] = value @@ -832,14 +900,16 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None: assert "stage_leoclean_proposal" in sql assert "'pending_review'" in sql assert "revoke all privileges on all tables in schema public, kb_stage" in sql - assert "can_insert_proposals_directly" in sql - assert "can_create_database_objects" in sql - assert "can_create_temp_objects" in sql - assert "can_connect_legacy_database" in sql + assert "grant insert" in sql.lower() assert "unexpectedly has database CREATE" in sql + assert "unexpectedly has schema CREATE" in sql + assert "runtime CONNECT reaches a noncanonical database" in sql + assert "PUBLIC retains database CONNECT" in sql + assert "PUBLIC retains large-object mutator EXECUTE" in sql assert "REVOKE TEMP FROM PUBLIC would be a" in sql assert "can execute unexpected SECURITY DEFINER functions" in sql - assert "revoke all privileges (%s) on table %I.%I from %I" in sql + assert "revoke connect on database %I from public" in sql + assert "revoke execute on function pg_catalog.%I(%s) from public" in sql assert re.search( r"grant\s+insert\s*\([^)]*proposal_type[^)]*payload[^)]*\)\s+" r"on\s+kb_stage\.kb_proposals\s+to\s+leoclean_kb_stage_owner", @@ -855,12 +925,13 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None: assert "grant delete" not in sql.lower() assert "create role leoclean_kb_runtime nologin" in sql.lower() disable_statement = "alter role leoclean_kb_runtime\n with nologin nosuperuser" - password_statement = "alter role leoclean_kb_runtime password" - assert sql.lower().index(disable_statement) < sql.lower().index(password_statement) - assert sql.lower().index(disable_statement) < sql.index("\\getenv runtime_password") - assert sql.index("\\getenv runtime_password") < sql.lower().index(password_statement) - assert sql.lower().index("with nologin nosuperuser") < sql.index("\\connect teleo_kb") - assert sql.index("\\connect teleo_kb") < sql.lower().rindex("with login nosuperuser") + password_statement = "\\password leoclean_kb_runtime" + assert sql.lower().index(disable_statement) < sql.index(password_statement) + assert "\\getenv runtime_password" not in sql + assert "password :'runtime_password'" not in sql.lower() + assert sql.lower().index("with nologin nosuperuser") < sql.index("\\connect teleo_canonical") + assert sql.index("\\connect teleo_canonical") < sql.lower().rindex("with login nosuperuser") + assert sql.lower().rindex("with login nosuperuser") < sql.rindex("commit;") assert "leoclean_kb_runtime final login attributes are unsafe" in sql @@ -869,6 +940,9 @@ def _load_module(path: Path): assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + if path.name == "cloudsql_memory_tool.py": + module.runtime_ssl_root_cert = lambda: module.RUNTIME_SSL_ROOT_CERT + module.runtime_cloudsdk_config = lambda: module.RUNTIME_CLOUDSDK_CONFIG return module diff --git a/tests/test_teleo_agent_systemd.py b/tests/test_teleo_agent_systemd.py index f6edaaa..1218a50 100644 --- a/tests/test_teleo_agent_systemd.py +++ b/tests/test_teleo_agent_systemd.py @@ -1,9 +1,33 @@ +import os +import resource +import signal import subprocess +import sys +import time from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[1] +def embedded_service_context_runner_source() -> str: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + marker = "remote_helpers=$(cat <<'REMOTE'\n" + start = script.index(marker) + len(marker) + end = script.index("\nREMOTE\n)", start) + helpers = script[start:end] + blocks = [part.split("\nPY", maxsplit=1)[0] for part in helpers.split("<<'PY'\n")[1:]] + return next(block for block in blocks if "def run_stopped_child(" in block) + + +def load_service_context_runner() -> dict[str, object]: + namespace: dict[str, object] = {"__name__": "embedded_service_context_runner_test"} + source = embedded_service_context_runner_source() + exec(compile(source, "", "exec"), namespace) + return namespace + + def test_teleo_agent_template_supports_optional_per_agent_env_file(): unit = (REPO_ROOT / "systemd" / "teleo-agent@.service").read_text() @@ -189,7 +213,7 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert "sha256sum" in script assert "--preflight-only" in script assert "--restart" in script - assert '/usr/bin/python3 "$tool_path" status --format json --redacted' in script + assert '/usr/bin/python3 -I "$tool_path" status --format json --redacted' in script assert "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" in script assert "169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" in script assert "ProxyHandler({})" in script @@ -204,7 +228,8 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): wrapper_probe = script.split("run_service_wrapper_status() {", maxsplit=1)[1].split( "\n}\n\nservice_fingerprint", maxsplit=1 )[0] - assert 'nsenter -t "$main_pid" -m --env' in wrapper_probe + assert 'run_in_service_cgroup "$main_pid"' in wrapper_probe + assert 'nsenter -t "$main_pid" -m -n --env' in wrapper_probe assert "/usr/bin/setpriv" in wrapper_probe assert "--reuid=teleo" in wrapper_probe assert "TELEO_CLOUDSQL_" not in wrapper_probe @@ -218,15 +243,54 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert "--property=EnvironmentFiles" in script assert "Service has an effective EnvironmentFile" in script assert "Service has a later drop-in" in script + assert "--property=PrivateNetwork" in script + assert "--property=PrivateUsers" in script + assert "NetworkNamespacePath" in script + assert "IPAddressDeny" in script + assert "IPAddressAllow" in script + assert "RestrictAddressFamilies" in script + assert "RestrictNetworkInterfaces" in script + assert "SystemCallFilter" in script + assert "AppArmorProfile" in script + assert "SELinuxContext" in script + assert '--property="$network_property"' in script assert "assert_administrator_secret_denied" in script assert '"stdout_discarded": True' in script assert '"credential_context": "systemd_main_pid_environment"' in script administrator_probe = script.split("assert_administrator_secret_denied() {", maxsplit=1)[1].split( "\n}\n\nrun_scoped_status", maxsplit=1 )[0] - assert 'nsenter -t "$main_pid" -m --env' in administrator_probe + assert 'run_in_service_cgroup "$main_pid"' in administrator_probe + assert 'nsenter -t "$main_pid" -m -n --env' in administrator_probe assert "/usr/bin/setpriv" in administrator_probe assert "env -i" not in administrator_probe + permission_probe = script.split("run_permission_verifier() {", maxsplit=1)[1].split( + "\n}\n\nassert_trusted_executable", maxsplit=1 + )[0] + assert 'if [ "$phase" = pre ]; then' in permission_probe + assert 'run_in_service_cgroup "$main_pid"' in permission_probe + assert 'nsenter -t "$main_pid" -m -n' in permission_probe + assert "/usr/bin/env -i" in permission_probe + assert 'verification_tmp="$(sudo mktemp -d /run/' in script + assert "run_in_service_cgroup() {" in script + assert "--property=ControlGroup" in script + assert 'Path(f"/proc/{pid}/cgroup")' in script + assert 'Path("/sys/fs/cgroup").resolve(strict=True)' in script + assert 'cgroup_dir / "cgroup.procs"' in script + assert "os.WUNTRACED" in script + assert "signal.SIGSTOP" in script + assert "signal.SIGCONT" in script + assert "os.waitpid(child, 0)" in script + assert "PR_SET_PDEATHSIG" in script + assert "signal.pthread_sigmask" in script + assert "ParentInterrupted" in script + assert "kill_and_reap(child)" in script + assert "Seccomp_filters" in script + assert '"/proc/$main_pid/attr/current"' in script + assert "/proc/1/ns/user" in script + assert "resource.prlimit" in script + assert "copy_process_limits" in script + assert "configure_stopped_child" in script assert 'preflight_service_before="$(service_fingerprint)"' in script assert 'assert_service_fingerprint "$preflight_service_before"' in script assert "backup_path" in script @@ -253,7 +317,7 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert 'REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"' in script assert 'sudo install -d -o root -g root -m 0755 "$runtime_parent"' in script assert 'sudo install -d -o root -g root -m 0755 "$runtime_bin"' in script - assert 'remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)"' in script + assert 'remote_tmp="$(sudo mktemp -d /run/teleo-gcp-leoclean-runtime.XXXXXX)"' in script assert 'sudo install -d -o root -g root -m 0755 "$payload_dir"' in script assert 'sudo install -d -o root -g root -m 0700 "$backup_dir"' in script assert 'sudo install -d -o teleo -g teleo -m 0700 "$work_dir"' in script @@ -288,7 +352,10 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert 'for executable_dir in \\\n /usr/local/sbin \\' in script assert 'sudo stat -Lc \'%U:%G\' "$executable_dir"' in script assert 'sudo -n -u teleo test ! -w "$executable_dir"' in script - assert 'for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3' in script + assert ( + 'for protected_file in /bin/bash /usr/bin/curl /usr/bin/env /usr/bin/gcloud ' + '/usr/bin/psql /usr/bin/python3 /usr/bin/setpriv' in script + ) assert '/usr/bin/curl -q --noproxy' in script assert 'test "$(sudo cat "$revision")" = "$SOURCE_REVISION"' in script assert 'assert_existing_root_directory "$existing_directory"' in script @@ -314,30 +381,39 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert 'sudo install -o root -g root -m 0755 "$work_dir/' not in script assert 'nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb"' in script assert 'findmnt -T "$SERVICE_PROFILE_BIN"' in script - assert 'findmnt -M "$SERVICE_HERMES_ROOT"' in script + assert "findmnt -M /home/teleo" in script assert 'findmnt -M "$SERVICE_PROFILE_ROOT/state"' in script assert 'findmnt -M "$SERVICE_PROFILE_ROOT/workspace"' in script - assert "cap_sys_admin" in script + assert "for capability_field in CapInh CapPrm CapEff CapBnd CapAmb" in script + assert "0000000000000000" in script assert '"current_tier": "T3_live_readonly"' in script assert '"status": "pass"' in script assert "UnsetEnvironment=PGPASSWORD" in dropin - assert "UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG" in dropin + assert ( + "UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CORE_PROJECT " + "CLOUDSDK_AUTH_ACCESS_TOKEN" in dropin + ) assert "UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS" in dropin assert "UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT" in dropin assert "UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT" in dropin assert "UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY" in dropin assert "UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE" in dropin + assert ( + "UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL " + "TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT" in dropin + ) assert ( "BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin" in dropin ) - assert "ReadOnlyPaths=/home/teleo/.hermes" in dropin + assert "ReadOnlyPaths=/home/teleo" in dropin assert ( "ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state " "/home/teleo/.hermes/profiles/leoclean/workspace" in dropin ) - assert "CapabilityBoundingSet=~CAP_SYS_ADMIN" in dropin + assert "CapabilityBoundingSet=" in dropin + assert "AmbientCapabilities=" in dropin assert "RuntimeDirectory=" not in dropin - assert "Environment=CLOUDSDK_" not in dropin + assert "Environment=CLOUDSDK_CONFIG=/usr/local/libexec/livingip/leoclean-kb/gcloud-config" in dropin assert "Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" in dropin assert "TELEO_KB_MODE=cloudsql" in dropin assert "TELEO_GCP_METADATA_ONLY=1" in dropin @@ -374,7 +450,7 @@ def test_gcp_runtime_embedded_remote_programs_are_bash_syntax_valid() -> None: assert completed.returncode == 0, f"{name}: {completed.stderr}" embedded_python = [part.split("\nPY", maxsplit=1)[0] for part in blocks["remote_helpers"].split("<<'PY'\n")[1:]] - assert len(embedded_python) == 2 + assert len(embedded_python) == 4 for index, body in enumerate(embedded_python): completed = subprocess.run( ["python3", "-c", "import sys; compile(sys.stdin.read(), '', 'exec')"], @@ -386,6 +462,162 @@ def test_gcp_runtime_embedded_remote_programs_are_bash_syntax_valid() -> None: assert completed.returncode == 0, f"embedded_python_{index}: {completed.stderr}" +@pytest.mark.parametrize( + ("command", "expected_returncode"), + [ + (["/bin/sh", "-c", "exit 0"], 0), + (["/bin/sh", "-c", "exit 7"], 7), + (["/bin/sh", "-c", "kill -TERM $$"], 128 + signal.SIGTERM), + ], +) +def test_service_context_runner_propagates_result_and_reaps_child( + command: list[str], + expected_returncode: int, +) -> None: + run_stopped_child = load_service_context_runner()["run_stopped_child"] + moved_children: list[int] = [] + + result = run_stopped_child( # type: ignore[operator] + command, + "/test.service", + move_child=moved_children.append, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + ) + + assert result == expected_returncode + assert len(moved_children) == 1 + with pytest.raises(ProcessLookupError): + os.kill(moved_children[0], 0) + + +def test_service_context_runner_reaps_stopped_child_when_move_fails() -> None: + run_stopped_child = load_service_context_runner()["run_stopped_child"] + stopped_children: list[int] = [] + + def fail_move(child: int) -> None: + stopped_children.append(child) + raise RuntimeError("synthetic move failure") + + with pytest.raises(RuntimeError, match="synthetic move failure"): + run_stopped_child( # type: ignore[operator] + ["/bin/sh", "-c", "exit 0"], + "/test.service", + move_child=fail_move, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + ) + + assert len(stopped_children) == 1 + with pytest.raises(ProcessLookupError): + os.kill(stopped_children[0], 0) + + +@pytest.mark.skipif(not hasattr(resource, "prlimit"), reason="requires Linux prlimit") +def test_service_context_runner_copies_mainpid_resource_limits(tmp_path: Path) -> None: + namespace = load_service_context_runner() + run_stopped_child = namespace["run_stopped_child"] + copy_process_limits = namespace["copy_process_limits"] + limit_readback = tmp_path / "limits.txt" + child_program = ( + "import resource,sys; from pathlib import Path; " + "limits=resource.getrlimit(resource.RLIMIT_NOFILE); " + "Path(sys.argv[1]).write_text(f'{limits[0]}:{limits[1]}', encoding='ascii')" + ) + + def lower_nofile_limit() -> None: + resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) + + source = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)"], + preexec_fn=lower_nofile_limit, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + result = run_stopped_child( # type: ignore[operator] + [sys.executable, "-c", child_program, str(limit_readback)], + "/test.service", + move_child=lambda _child: None, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + configure_stopped_child=lambda child: copy_process_limits(source.pid, child), # type: ignore[operator] + ) + finally: + source.terminate() + source.wait(timeout=5) + + assert result == 0 + assert limit_readback.read_text(encoding="ascii") == "64:64" + + +@pytest.mark.parametrize("signum", [signal.SIGTERM, signal.SIGHUP]) +def test_service_context_runner_interruption_kills_and_reaps_probe( + tmp_path: Path, + signum: signal.Signals, +) -> None: + run_stopped_child = load_service_context_runner()["run_stopped_child"] + child_pid_path = tmp_path / f"probe-{signum.name}.pid" + child_program = ( + "import os,sys,time; from pathlib import Path; " + "Path(sys.argv[1]).write_text(str(os.getpid()), encoding='ascii'); " + "time.sleep(30)" + ) + runner_pid = os.fork() + if runner_pid == 0: + result = run_stopped_child( # type: ignore[operator] + [sys.executable, "-c", child_program, str(child_pid_path)], + "/test.service", + move_child=lambda _child: None, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + ) + os._exit(result) + + runner_reaped = False + probe_pid: int | None = None + try: + deadline = time.monotonic() + 5 + probe_pid_text = "" + while time.monotonic() < deadline: + if child_pid_path.is_file(): + probe_pid_text = child_pid_path.read_text(encoding="ascii") + if probe_pid_text.isdecimal(): + break + waited, _status = os.waitpid(runner_pid, os.WNOHANG) + if waited == runner_pid: + runner_reaped = True + pytest.fail("service-context runner exited before probe start") + time.sleep(0.01) + assert probe_pid_text.isdecimal() + probe_pid = int(probe_pid_text) + + os.kill(runner_pid, signum) + waited, status = os.waitpid(runner_pid, 0) + runner_reaped = True + assert waited == runner_pid + assert os.WIFSIGNALED(status) + assert os.WTERMSIG(status) == signum + with pytest.raises(ProcessLookupError): + os.kill(probe_pid, 0) + finally: + if not runner_reaped: + try: + os.kill(runner_pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.waitpid(runner_pid, 0) + except ChildProcessError: + pass + if probe_pid is not None: + try: + os.kill(probe_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_gcp_runtime_rejects_preexisting_symlink_before_mutation(tmp_path: Path) -> None: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() function_start = script.index("assert_existing_root_directory() {") @@ -428,10 +660,43 @@ set -euo pipefail SERVICE=leoclean.service REMOTE_DROPIN_DIR=/etc/systemd/system/leoclean.service.d DROPIN_NAME=10-cloudsql-memory.conf +REMOTE_RUNTIME_BIN=/usr/local/libexec/livingip/leoclean-kb +SERVICE_PROFILE_ROOT=/home/teleo/.hermes/profiles/leoclean +SERVICE_PROFILE_BIN=$SERVICE_PROFILE_ROOT/bin +assert_exact_word_set() {{ + local actual="$1" expected="$2" label="$3" actual_sorted expected_sorted word + actual_sorted="$(for word in $actual; do printf '%s\\n' "${{word#-}}"; done | sort)" + expected_sorted="$(for word in $expected; do printf '%s\\n' "${{word#-}}"; done | sort)" + [ "$actual_sorted" = "$expected_sorted" ] || {{ + echo "Service effective $label differs from the reviewed exact allowlist." >&2 + return 1 + }} +}} systemctl() {{ - case "$*" in - *--property=DropInPaths*) printf '%s\\n' "${{FAKE_DROPINS:-}}" ;; - *--property=EnvironmentFiles*) printf '%s\\n' "${{FAKE_ENVIRONMENT_FILES:-}}" ;; + local argument property= + for argument in "$@"; do + case "$argument" in --property=*) property=${{argument#--property=}} ;; esac + done + case "$property" in + DropInPaths) printf '%s\\n' "${{FAKE_DROPINS:-}}" ;; + EnvironmentFiles) printf '%s\\n' "${{FAKE_ENVIRONMENT_FILES:-}}" ;; + LoadCredential|LoadCredentialEncrypted|SetCredential|SetCredentialEncrypted|ImportCredential|ExecCondition|ExecStartPre|ExecStartPost|ExecReload|ExecStop|ExecStopPost) + local variable="FAKE_$property" + printf '%s\\n' "${{!variable:-}}" + ;; + CapabilityBoundingSet|AmbientCapabilities|BindPaths|TemporaryFileSystem) printf '\\n' ;; + NoNewPrivileges) printf 'yes\\n' ;; + PrivateNetwork) printf '%s\\n' "${{FAKE_PrivateNetwork:-no}}" ;; + PrivateUsers) printf '%s\\n' "${{FAKE_PrivateUsers:-no}}" ;; + NetworkNamespacePath|IPAddressDeny|IPAddressAllow|RestrictAddressFamilies|RestrictNetworkInterfaces|SocketBindAllow|SocketBindDeny|SystemCallFilter|AppArmorProfile|SELinuxContext) + local variable="FAKE_$property" + printf '%s\\n' "${{!variable:-}}" + ;; + Group|SupplementaryGroups) printf 'teleo\\n' ;; + ReadOnlyPaths) printf '%s\\n' "${{FAKE_ReadOnlyPaths:-/home/teleo}}" ;; + ReadWritePaths) printf '%s\\n' "$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" ;; + BindReadOnlyPaths) printf '%s\\n' "$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" ;; + InaccessiblePaths) printf '%s\\n' '/home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql' ;; *) return 90 ;; esac }} @@ -445,6 +710,19 @@ assert_unit_configuration "FAKE_ENVIRONMENT_FILES": "", }, {"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": "/etc/leoclean-runtime.env"}, + {"FAKE_DROPINS": reviewed, "FAKE_ExecStartPost": "/usr/local/bin/conflicting-hook"}, + {"FAKE_DROPINS": reviewed, "FAKE_ReadOnlyPaths": "/home/teleo /tmp"}, + {"FAKE_DROPINS": reviewed, "FAKE_PrivateNetwork": "yes"}, + {"FAKE_DROPINS": reviewed, "FAKE_PrivateUsers": "yes"}, + {"FAKE_DROPINS": reviewed, "FAKE_NetworkNamespacePath": "/run/netns/restricted"}, + {"FAKE_DROPINS": reviewed, "FAKE_IPAddressDeny": "any"}, + {"FAKE_DROPINS": reviewed, "FAKE_IPAddressAllow": "10.61.0.4/32"}, + {"FAKE_DROPINS": reviewed, "FAKE_RestrictAddressFamilies": "AF_UNIX"}, + {"FAKE_DROPINS": reviewed, "FAKE_RestrictNetworkInterfaces": "~ens4"}, + {"FAKE_DROPINS": reviewed, "FAKE_SocketBindDeny": "any"}, + {"FAKE_DROPINS": reviewed, "FAKE_SystemCallFilter": "@system-service ~@network-io"}, + {"FAKE_DROPINS": reviewed, "FAKE_AppArmorProfile": "restricted-profile"}, + {"FAKE_DROPINS": reviewed, "FAKE_SELinuxContext": "system_u:system_r:restricted_t:s0"}, ) for environment in cases: completed = subprocess.run( @@ -462,12 +740,53 @@ assert_unit_configuration input=harness, capture_output=True, check=False, - env={"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": ""}, + env={"FAKE_DROPINS": reviewed}, text=True, ) assert allowed.returncode == 0, allowed.stderr +def test_gcp_runtime_effective_execstart_override_is_rejected_before_process_probe() -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("assert_service_running() {") + function_end = script.index("\n\nassert_unit_configuration() {", function_start) + service_function = script[function_start:function_end] + harness = f""" +set -euo pipefail +SERVICE=leoclean.service +EXPECTED_HERMES_EXECUTABLE=/home/teleo/.hermes/hermes-agent/hermes.py +EXPECTED_HERMES_PYTHON=/home/teleo/.hermes/hermes-agent/venv/bin/python +systemctl() {{ + local argument property= + for argument in "$@"; do + case "$argument" in --property=*) property=${{argument#--property=}} ;; esac + done + case "$property" in + ActiveState) printf 'active\\n' ;; + SubState) printf 'running\\n' ;; + MainPID) printf '1234\\n' ;; + User) printf 'teleo\\n' ;; + ExecStart) printf '{{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /tmp/conflicting.py ; }}\\n' ;; + *) return 90 ;; + esac +}} +ps() {{ printf 'teleo\\n'; }} +{service_function} +assert_service_running false +""" + + completed = subprocess.run( + ["bash", "-s"], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode != 0 + assert "effective ExecStart is not the reviewed" in completed.stderr + + def test_gcp_runtime_absence_rollback_removes_every_created_directory(tmp_path: Path) -> None: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() function_start = script.index("backup_directory_state() {") From f79c779f4904882d0b8ba97c37081f4695cb8061 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 07:21:23 +0200 Subject: [PATCH 4/7] ci: cover GCP runtime security surfaces --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3367127..fc748bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,7 @@ jobs: scripts/replay_decision_engine_eval.py \ scripts/prove_phase1b_local.py \ scripts/run_gcp_generated_db_direct_claim_suite.py \ + scripts/run_gcp_generated_db_working_leo_suite.py \ scripts/run_gcp_generated_db_blind_claim_canary.py \ scripts/run_leo_direct_claim_handler_suite.py \ scripts/run_leo_m3taversal_oos_handler_suite.py \ @@ -92,6 +93,7 @@ jobs: tests/test_gcp_iam_split_plan.py \ tests/test_gcp_readiness_workflow.py \ tests/test_gcp_generated_db_direct_claim_suite.py \ + tests/test_gcp_generated_db_working_leo_suite.py \ tests/test_gcp_generated_db_blind_claim_canary.py \ tests/test_hermes_leoclean_kb_bridge_source.py \ tests/test_hermes_leoclean_skill_surfaces.py \ @@ -115,7 +117,9 @@ jobs: bash -n \ deploy/sync-gcp-leoclean-runtime.sh \ hermes-agent/leoclean-bin/teleo-kb \ + hermes-agent/leoclean-bin/teleo-kb-gcp \ ops/backup_vps_sqlite_kb.sh \ + ops/provision_gcp_leoclean_runtime_role.sh \ ops/run_gcp_cloudsql_restore_drill.sh \ ops/run_sqlite_postgres_restore_canary.sh From 25c26824765187ad7e6829daf8edf2ee06b620f6 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 07:31:37 +0200 Subject: [PATCH 5/7] test: isolate runtime guard from CI environment --- .../test_hermes_leoclean_kb_bridge_source.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index 90105f1..5eaa0ce 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -308,7 +308,20 @@ def _configure_runtime_environment(monkeypatch: pytest.MonkeyPatch) -> None: normalized_key.startswith("PG") or normalized_key.startswith("CLOUDSDK_") or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS" - or normalized_key in {"LD_PRELOAD", "PYTHONHOME", "PYTHONPATH"} + or normalized_key.startswith("LD_") + or normalized_key.startswith("BASH_FUNC_") + or normalized_key.startswith("PYTHON") + or normalized_key + in { + "BASH_ENV", + "BASHOPTS", + "CDPATH", + "ENV", + "GLOBIGNORE", + "IFS", + "PS4", + "SHELLOPTS", + } or normalized_key == "PROXY" or normalized_key.endswith("_PROXY") or normalized_key in { @@ -325,6 +338,24 @@ def _configure_runtime_environment(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") +def test_runtime_environment_fixture_scrubs_github_runner_inheritance( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + monkeypatch.setenv("CI", "true") + monkeypatch.setenv("LD_LIBRARY_PATH", "/opt/hostedtoolcache/Python/3.11.15/x64/lib") + monkeypatch.setenv("pythonLocation", "/opt/hostedtoolcache/Python/3.11.15/x64") + monkeypatch.setenv("HTTP_PROXY", "http://runner-proxy.invalid:8080") + monkeypatch.setenv("BASH_FUNC_runner%%", "() { :; }") + monkeypatch.setenv("SSL_CERT_FILE", "/tmp/runner-ca.pem") + + _configure_runtime_environment(monkeypatch) + + assert os.environ["CI"] == "true" + module.validate_runtime_connection(args) + + class _FakeHttpResponse: def __init__(self, body: bytes, status: int = 200) -> None: self.body = body From 1b21139d4e22396ec8f93c9c362181d964cadaa4 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 07:35:59 +0200 Subject: [PATCH 6/7] test: mirror runtime trust environment --- tests/test_hermes_leoclean_kb_bridge_source.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index 5eaa0ce..6022d2a 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -331,6 +331,10 @@ def _configure_runtime_environment(monkeypatch: pytest.MonkeyPatch) -> None: "SSL_CERT_DIR", "SSL_CERT_FILE", "SSLKEYLOGFILE", + "OPENSSL_CONF", + "OPENSSL_ENGINES", + "OPENSSL_MODULES", + "GCONV_PATH", } ): monkeypatch.delenv(key, raising=False) @@ -349,6 +353,8 @@ def test_runtime_environment_fixture_scrubs_github_runner_inheritance( monkeypatch.setenv("HTTP_PROXY", "http://runner-proxy.invalid:8080") monkeypatch.setenv("BASH_FUNC_runner%%", "() { :; }") monkeypatch.setenv("SSL_CERT_FILE", "/tmp/runner-ca.pem") + monkeypatch.setenv("OPENSSL_CONF", "/tmp/runner-openssl.cnf") + monkeypatch.setenv("GCONV_PATH", "/tmp/runner-gconv") _configure_runtime_environment(monkeypatch) From 959ba3cb7b686ed9d1d34cc1b3a03fab4f4ee463 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 09:51:51 +0200 Subject: [PATCH 7/7] test: wait for Linux service environment readiness --- .../test_gcp_leoclean_service_environment.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_gcp_leoclean_service_environment.py b/tests/test_gcp_leoclean_service_environment.py index 449ed8d..57c37ae 100644 --- a/tests/test_gcp_leoclean_service_environment.py +++ b/tests/test_gcp_leoclean_service_environment.py @@ -2,6 +2,7 @@ import json import os import subprocess import sys +import time from collections.abc import Mapping from pathlib import Path @@ -14,6 +15,27 @@ def environment_block(environment: Mapping[str, str]) -> bytes: return b"\0".join(f"{key}={value}".encode() for key, value in environment.items()) + b"\0" +def wait_for_process_environment( + process: subprocess.Popen[bytes], + expected: Mapping[str, str], + *, + timeout_seconds: float = 5.0, +) -> None: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if process.poll() is not None: + pytest.fail("environment fixture process exited before becoming readable") + try: + observed = verifier.parse_environment(Path(f"/proc/{process.pid}/environ").read_bytes()) + except (OSError, verifier.VerificationError): + time.sleep(0.01) + continue + if observed == expected: + return + time.sleep(0.01) + pytest.fail("environment fixture process did not become readable") + + def test_all_expected_fields_pass_and_receipt_contains_no_values() -> None: environment = verifier.expected_environment() environment.update({"HOME": "/home/teleo", "LANG": "C.UTF-8"}) @@ -234,6 +256,7 @@ def test_cli_reads_real_process_environment_and_detects_effective_conflict( stderr=subprocess.DEVNULL, ) try: + wait_for_process_environment(good_process, expected) assert verifier.main(["--pid", str(good_process.pid)]) == 0 good_output = capsys.readouterr() assert good_output.err == "" @@ -253,6 +276,7 @@ def test_cli_reads_real_process_environment_and_detects_effective_conflict( stderr=subprocess.DEVNULL, ) try: + wait_for_process_environment(bad_process, conflicting) assert verifier.main(["--pid", str(bad_process.pid)]) == 1 bad_output = capsys.readouterr() receipt = json.loads(bad_output.out)