diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c069e36..9b70be5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,8 @@ jobs: ops/restore_gcp_generated_postgres_snapshot.py \ ops/sqlite_to_postgres_dump.py \ ops/verify_gcp_cloudsql_restore_readback.py \ + ops/verify_gcp_leoclean_runtime_permissions.py \ + ops/verify_gcp_leoclean_service_environment.py \ ops/verify_gcp_canonical_lifecycle.py \ ops/verify_postgres_parity_manifest.py \ ops/verify_vps_canonical_snapshot_delta.py \ @@ -69,6 +71,7 @@ jobs: scripts/replay_decision_engine_eval.py \ scripts/prove_phase1b_local.py \ scripts/run_gcp_generated_db_direct_claim_suite.py \ + scripts/run_gcp_generated_db_working_leo_suite.py \ scripts/run_gcp_generated_db_blind_claim_canary.py \ scripts/run_leo_direct_claim_handler_suite.py \ scripts/run_leo_m3taversal_oos_handler_suite.py \ @@ -86,6 +89,8 @@ jobs: tests/test_capture_vps_canonical_postgres_snapshot.py \ tests/test_gcp_infra_execute_canary.py \ tests/test_gcp_infra_readiness_checker.py \ + tests/test_gcp_leoclean_runtime_permissions.py \ + tests/test_gcp_leoclean_service_environment.py \ tests/test_gcp_runtime_baseline_apply.py \ tests/test_gcp_service_communications.py \ tests/test_gcp_cloudsql_restore_drill.py \ @@ -94,6 +99,7 @@ jobs: tests/test_gcp_iam_split_plan.py \ tests/test_gcp_readiness_workflow.py \ tests/test_gcp_generated_db_direct_claim_suite.py \ + tests/test_gcp_generated_db_working_leo_suite.py \ tests/test_gcp_generated_db_blind_claim_canary.py \ tests/test_hermes_leoclean_kb_bridge_source.py \ tests/test_hermes_leoclean_skill_surfaces.py \ @@ -113,13 +119,18 @@ jobs: tests/test_run_local_genesis_ledger_rebuild.py \ tests/test_sqlite_to_postgres_dump.py \ tests/test_sqlite_postgres_restore_canary_capsule.py \ + tests/test_teleo_agent_systemd.py \ tests/test_eval_parse.py \ tests/test_contributor.py \ tests/test_search.py - name: Shell syntax run: | bash -n \ + deploy/sync-gcp-leoclean-runtime.sh \ + hermes-agent/leoclean-bin/teleo-kb \ + hermes-agent/leoclean-bin/teleo-kb-gcp \ ops/backup_vps_sqlite_kb.sh \ + ops/provision_gcp_leoclean_runtime_role.sh \ ops/run_gcp_cloudsql_restore_drill.sh \ ops/run_sqlite_postgres_restore_canary.sh diff --git a/deploy/sync-gcp-leoclean-runtime.sh b/deploy/sync-gcp-leoclean-runtime.sh new file mode 100755 index 0000000..b0edacb --- /dev/null +++ b/deploy/sync-gcp-leoclean-runtime.sh @@ -0,0 +1,1430 @@ +#!/usr/bin/env bash +# Deploy the source-controlled Cloud SQL bridge and non-secret service settings +# to the GCP parallel leoclean runtime. This does not create database users, +# read secret values, send Telegram messages, or promote Cloud SQL. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PROJECT="${PROJECT:-teleo-501523}" +ZONE="${ZONE:-europe-west6-a}" +INSTANCE="${INSTANCE:-teleo-prod-1}" +SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}" +MERGED_REF="${MERGED_REF:-origin/main}" +REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}" +SERVICE_PROFILE_BIN="${SERVICE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}" +SERVICE_PROFILE_ROOT="${SERVICE_PROFILE_ROOT:-/home/teleo/.hermes/profiles/leoclean}" +SERVICE_HERMES_ROOT="${SERVICE_HERMES_ROOT:-/home/teleo/.hermes}" +REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}" +EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}" +EXPECTED_HERMES_EXECUTABLE="${EXPECTED_HERMES_EXECUTABLE:-/home/teleo/.hermes/hermes-agent/venv/bin/hermes}" +EXPECTED_HERMES_PYTHON="${EXPECTED_HERMES_PYTHON:-/home/teleo/.hermes/hermes-agent/venv/bin/python3}" + +WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb-gcp" +TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py" +PERMISSION_VERIFIER_REL="ops/verify_gcp_leoclean_runtime_permissions.py" +SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py" +SERVER_CA_REL="ops/gcp-teleo-pgvector-standby-server-ca.pem" +DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf" +DROPIN_NAME="10-cloudsql-memory.conf" + +DRY_RUN=false +RESTART=false +VERIFY_ONLY=false +PREFLIGHT_ONLY=false + +usage() { + cat <<'USAGE' +Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--preflight-only] [--restart] [--verify-only] + +The database role and scoped Secret Manager secret must exist before --restart. +This script deploys only the bridge files and non-secret systemd drop-in. +USAGE +} + +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --preflight-only) PREFLIGHT_ONLY=true ;; + --restart) RESTART=true ;; + --verify-only) VERIFY_ONLY=true ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown arg: $arg" >&2 + usage >&2 + exit 64 + ;; + esac +done + +if $PREFLIGHT_ONLY && $RESTART; then + echo "--preflight-only and --restart are mutually exclusive." >&2 + exit 64 +fi +if $VERIFY_ONLY && { $PREFLIGHT_ONLY || $RESTART; }; then + echo "--verify-only cannot be combined with --preflight-only or --restart." >&2 + exit 64 +fi + +for path in \ + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ + "$DROPIN_REL"; do + if [ ! -f "$REPO_ROOT/$path" ]; then + echo "Required source file is missing: $path" >&2 + exit 66 + fi + if [ -L "$REPO_ROOT/$path" ]; then + echo "Refusing symlinked runtime source file: $path" >&2 + exit 65 + fi +done + +if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2 + exit 65 +fi +if ! grep -Fxq 'Environment=TELEO_GCP_METADATA_ONLY=1' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must enforce metadata-only GCP credentials." >&2 + exit 65 +fi +if ! grep -Fxq "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not bind the root-controlled runtime into the service profile." >&2 + exit 65 +fi +if ! grep -Fxq 'ReadOnlyPaths=/home/teleo' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not make the complete service home read-only." >&2 + exit 65 +fi +if ! grep -Fxq "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not limit service-profile writes to state and workspace." >&2 + exit 65 +fi +if ! grep -Fxq 'CapabilityBoundingSet=' "$REPO_ROOT/$DROPIN_REL" || + ! grep -Fxq 'AmbientCapabilities=' "$REPO_ROOT/$DROPIN_REL" || + ! grep -Fxq 'NoNewPrivileges=yes' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must remove all capabilities and enforce no-new-privileges." >&2 + exit 65 +fi +if ! grep -Fxq 'Group=teleo' "$REPO_ROOT/$DROPIN_REL" || + ! grep -Fxq 'SupplementaryGroups=teleo' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must bind the exact service group set." >&2 + exit 65 +fi +if ! grep -Fxq 'UnsetEnvironment=OPENSSL_CONF OPENSSL_MODULES OPENSSL_ENGINES GCONV_PATH CREDENTIALS_DIRECTORY' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must remove startup-loader and systemd-credential injection fields." >&2 + exit 65 +fi +if ! grep -Fxq 'InaccessiblePaths=-/home/teleo/.config/gcloud -/home/teleo/.pgpass -/home/teleo/.pg_service.conf -/home/teleo/.postgresql' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must mask default GCP and PostgreSQL credential paths." >&2 + exit 65 +fi +if ! grep -Fxq \ + 'UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT' \ + "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must unset optional Cloud SQL and memory controls." >&2 + exit 65 +fi + +SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)" +if ! git -C "$REPO_ROOT" diff --quiet HEAD -- \ + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ + "$DROPIN_REL"; then + echo "Refusing to deploy runtime files that do not match HEAD." >&2 + exit 65 +fi +if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- \ + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ + "$DROPIN_REL")" ]; then + echo "Refusing to deploy untracked runtime files." >&2 + exit 65 +fi +if ! git -C "$REPO_ROOT" rev-parse --verify "$MERGED_REF^{commit}" >/dev/null 2>&1; then + echo "Merged reference is unavailable: $MERGED_REF" >&2 + exit 69 +fi +if ! git -C "$REPO_ROOT" merge-base --is-ancestor "$SOURCE_REVISION" "$MERGED_REF"; then + echo "Refusing to deploy unmerged revision $SOURCE_REVISION (not contained in $MERGED_REF)." >&2 + exit 65 +fi +WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')" +TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')" +PERMISSION_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" +SERVICE_ENV_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" +SERVER_CA_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVER_CA_REL" | awk '{print $1}')" +DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')" + +GCP_SSH=( + gcloud compute ssh "$INSTANCE" + --project="$PROJECT" + --zone="$ZONE" + --tunnel-through-iap +) + +remote_helpers=$(cat <<'REMOTE' +assert_exact_word_set() { + local actual="$1" expected="$2" label="$3" actual_sorted expected_sorted word + actual_sorted="$(for word in $actual; do printf '%s\n' "${word#-}"; done | sort)" + expected_sorted="$(for word in $expected; do printf '%s\n' "${word#-}"; done | sort)" + if [ "$actual_sorted" != "$expected_sorted" ]; then + echo "Service effective $label differs from the reviewed exact allowlist." >&2 + return 1 + fi +} + +assert_service_running() { + local require_reviewed="${1:-true}" + local active_state sub_state main_pid unit_user process_user teleo_uid teleo_gid + local uid_fields gid_fields groups_field exec_start process_executable + local seccomp_mode seccomp_filters lsm_context + local host_user_namespace service_user_namespace + active_state="$(systemctl show "$SERVICE" --property=ActiveState --value)" + sub_state="$(systemctl show "$SERVICE" --property=SubState --value)" + if [ "$active_state" != "active" ] || [ "$sub_state" != "running" ]; then + echo "Service is not active/running: ActiveState=$active_state SubState=$sub_state" >&2 + return 1 + fi + + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + unit_user="$(systemctl show "$SERVICE" --property=User --value)" + case "$main_pid" in + ''|0|*[!0-9]*) + echo "Service has no live MainPID: $main_pid" >&2 + return 1 + ;; + esac + if [ "$unit_user" != "teleo" ]; then + echo "Service unit user is not teleo: $unit_user" >&2 + return 1 + fi + process_user="$(ps -o user= -p "$main_pid" | awk '{print $1}')" + if [ "$process_user" != "teleo" ]; then + echo "Service process user is not teleo: $process_user" >&2 + return 1 + fi + if [ "$require_reviewed" = true ]; then + teleo_uid="$(id -u teleo)" + teleo_gid="$(id -g teleo)" + uid_fields="$(awk '/^Uid:/{print $2 ":" $3 ":" $4 ":" $5}' "/proc/$main_pid/status")" + gid_fields="$(awk '/^Gid:/{print $2 ":" $3 ":" $4 ":" $5}' "/proc/$main_pid/status")" + groups_field="$(awk '/^Groups:/{for (i=2;i<=NF;i++) printf "%s%s", (i==2 ? "" : ":"), $i}' "/proc/$main_pid/status")" + if [ "$uid_fields" != "$teleo_uid:$teleo_uid:$teleo_uid:$teleo_uid" ] || + [ "$gid_fields" != "$teleo_gid:$teleo_gid:$teleo_gid:$teleo_gid" ] || + [ "$groups_field" != "$teleo_gid" ]; then + echo "Service process does not have the exact reviewed uid/gid/group set." >&2 + return 1 + fi + fi + exec_start="$(systemctl show "$SERVICE" --property=ExecStart --value)" + case "$exec_start" in + *"path=$EXPECTED_HERMES_EXECUTABLE ; argv[]=$EXPECTED_HERMES_EXECUTABLE -p leoclean gateway run ;"*) ;; + *) + echo "Service effective ExecStart is not the reviewed leoclean Hermes gateway." >&2 + return 1 + ;; + esac + sudo /usr/bin/python3 -I - "$main_pid" "$EXPECTED_HERMES_PYTHON" "$EXPECTED_HERMES_EXECUTABLE" <<'PY' +import os +import sys +from pathlib import Path + +pid, expected_python, expected_hermes = sys.argv[1:] +raw = Path(f"/proc/{pid}/cmdline").read_bytes() +if not raw.endswith(b"\0") or b"\0\0" in raw: + raise SystemExit("service cmdline is malformed") +try: + argv = [part.decode("utf-8", "strict") for part in raw[:-1].split(b"\0")] +except UnicodeDecodeError: + raise SystemExit("service cmdline is not valid UTF-8") from None +if argv != [expected_python, expected_hermes, "-p", "leoclean", "gateway", "run"]: + raise SystemExit("service cmdline is not the reviewed leoclean Hermes gateway") +if os.path.realpath(f"/proc/{pid}/exe") != os.path.realpath(expected_python): + raise SystemExit("service executable is not the reviewed Hermes interpreter") +PY + if [ "$require_reviewed" = true ]; then + for capability_field in CapInh CapPrm CapEff CapBnd CapAmb; do + if [ "$(awk -v key="$capability_field:" '$1 == key {print $2}' "/proc/$main_pid/status")" != 0000000000000000 ]; then + echo "Service process retains capabilities in $capability_field." >&2 + return 1 + fi + done + if [ "$(awk '/^NoNewPrivs:/{print $2}' "/proc/$main_pid/status")" != 1 ]; then + echo "Service process does not enforce no-new-privileges." >&2 + return 1 + fi + seccomp_mode="$(awk '/^Seccomp:/{print $2}' "/proc/$main_pid/status")" + seccomp_filters="$(awk '/^Seccomp_filters:/{print $2}' "/proc/$main_pid/status")" + lsm_context="$(sudo cat "/proc/$main_pid/attr/current")" + host_user_namespace="$(sudo stat -Lc '%d:%i' /proc/1/ns/user)" + service_user_namespace="$(sudo stat -Lc '%d:%i' "/proc/$main_pid/ns/user")" + if [ "$seccomp_mode" != 0 ] || + { [ -n "$seccomp_filters" ] && [ "$seccomp_filters" != 0 ]; } || + [ "$lsm_context" != unconfined ] || + [ "$service_user_namespace" != "$host_user_namespace" ]; then + echo "Service process has an unreviewed live seccomp, LSM, or user-namespace restriction." >&2 + return 1 + fi + fi +} + +assert_unit_configuration() { + local require_reviewed="${1:-true}" + local dropin_path dropin_name dropin_paths environment_files reviewed_dropin_seen=false + local property value capability_set ambient_capabilities no_new_privileges unit_group supplementary_groups + local inaccessible_paths credential_path + local read_only_paths read_write_paths bind_paths bind_read_only_paths temporary_file_systems + local private_network private_users network_property + dropin_paths="$(systemctl show "$SERVICE" --property=DropInPaths --value)" + environment_files="$(systemctl show "$SERVICE" --property=EnvironmentFiles --value)" + if [ -n "$environment_files" ]; then + echo "Service has an effective EnvironmentFile, which is not allowed for the scoped runtime." >&2 + return 1 + fi + for property in LoadCredential LoadCredentialEncrypted SetCredential SetCredentialEncrypted ImportCredential ExecCondition ExecStartPre ExecStartPost ExecReload ExecStop ExecStopPost; do + value="$(systemctl show "$SERVICE" --property="$property" --value)" + if [ -n "$value" ]; then + echo "Service has an effective $property credential source." >&2 + return 1 + fi + done + for dropin_path in $dropin_paths; do + if [ "$dropin_path" = "$REMOTE_DROPIN_DIR/$DROPIN_NAME" ]; then + reviewed_dropin_seen=true + continue + fi + dropin_name="${dropin_path##*/}" + if [[ "$dropin_name" > "$DROPIN_NAME" ]]; then + echo "Service has a later drop-in than the reviewed Cloud SQL configuration: $dropin_name" >&2 + return 1 + fi + done + if [ "$require_reviewed" = true ] && [ "$reviewed_dropin_seen" != true ]; then + echo "Service does not load the reviewed Cloud SQL drop-in." >&2 + return 1 + fi + if [ "$require_reviewed" = true ]; then + capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)" + ambient_capabilities="$(systemctl show "$SERVICE" --property=AmbientCapabilities --value)" + no_new_privileges="$(systemctl show "$SERVICE" --property=NoNewPrivileges --value)" + unit_group="$(systemctl show "$SERVICE" --property=Group --value)" + supplementary_groups="$(systemctl show "$SERVICE" --property=SupplementaryGroups --value)" + inaccessible_paths="$(systemctl show "$SERVICE" --property=InaccessiblePaths --value)" + read_only_paths="$(systemctl show "$SERVICE" --property=ReadOnlyPaths --value)" + read_write_paths="$(systemctl show "$SERVICE" --property=ReadWritePaths --value)" + bind_paths="$(systemctl show "$SERVICE" --property=BindPaths --value)" + bind_read_only_paths="$(systemctl show "$SERVICE" --property=BindReadOnlyPaths --value)" + temporary_file_systems="$(systemctl show "$SERVICE" --property=TemporaryFileSystem --value)" + if [ -n "$capability_set" ] || [ -n "$ambient_capabilities" ] || [ "$no_new_privileges" != yes ] || + [ "$unit_group" != teleo ] || [ "$supplementary_groups" != teleo ]; then + echo "Service effective capability, privilege, or group contract differs from the reviewed drop-in." >&2 + return 1 + fi + assert_exact_word_set "$read_only_paths" "/home/teleo" ReadOnlyPaths + assert_exact_word_set "$read_write_paths" "$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" ReadWritePaths + assert_exact_word_set "$bind_paths" "" BindPaths + assert_exact_word_set "$bind_read_only_paths" "$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" BindReadOnlyPaths + assert_exact_word_set "$temporary_file_systems" "" TemporaryFileSystem + assert_exact_word_set "$supplementary_groups" "teleo" SupplementaryGroups + assert_exact_word_set "$inaccessible_paths" "/home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql" InaccessiblePaths + private_network="$(systemctl show "$SERVICE" --property=PrivateNetwork --value)" + private_users="$(systemctl show "$SERVICE" --property=PrivateUsers --value)" + if [ "$private_network" != no ] || [ "$private_users" != no ]; then + echo "Service effective private network or user namespace differs from the reviewed host contract." >&2 + return 1 + fi + for network_property in \ + NetworkNamespacePath \ + IPAddressDeny \ + IPAddressAllow \ + RestrictAddressFamilies \ + RestrictNetworkInterfaces \ + SocketBindAllow \ + SocketBindDeny \ + SystemCallFilter \ + AppArmorProfile \ + SELinuxContext; do + value="$(systemctl show "$SERVICE" --property="$network_property" --value)" + if [ -n "$value" ]; then + echo "Service effective $network_property differs from the reviewed unrestricted transport contract." >&2 + return 1 + fi + done + for credential_path in /home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql; do + case " $inaccessible_paths " in + *" $credential_path "*) ;; + *) + echo "Service does not mask a default credential path: $credential_path" >&2 + return 1 + ;; + esac + done + fi +} + +assert_runtime_environment() { + local verifier_path="$1" require_reviewed="${2:-true}" main_pid + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + if [ "$require_reviewed" = true ]; then + sudo /usr/bin/python3 -I "$verifier_path" --pid "$main_pid" + else + sudo /usr/bin/python3 -I "$verifier_path" --pid "$main_pid" --allow-missing-runtime-security-fields + fi + assert_unit_configuration "$require_reviewed" + if [ "$require_reviewed" = true ]; then + sudo nsenter -t "$main_pid" -m -- test ! -e "/run/credentials/$SERVICE" + for credential_path in /home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql; do + sudo nsenter -t "$main_pid" -m -- \ + /usr/bin/setpriv --reuid=teleo --regid=teleo --clear-groups --no-new-privs \ + /usr/bin/test ! -r "$credential_path" + done + fi +} + +assert_attached_service_account() { + local metadata_email + metadata_email="$( + sudo -n -u teleo env -i \ + HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + /usr/bin/curl -q --noproxy '*' --proto '=http' --max-time 10 --fail --silent --show-error \ + --header 'Metadata-Flavor: Google' \ + 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email' + )" + if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then + echo "Attached VM service account mismatch: $metadata_email" >&2 + return 1 + fi + echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email" +} + +run_in_service_cgroup() { + local main_pid="$1" control_group + shift + control_group="$(systemctl show "$SERVICE" --property=ControlGroup --value)" + if [ -z "$control_group" ]; then + echo "Service has no effective cgroup." >&2 + return 1 + fi + sudo /usr/bin/python3 -I - "$main_pid" "$control_group" "$@" <<'PY' +import ctypes +import os +import resource +import signal +import stat +import sys +from pathlib import Path, PurePosixPath + + +HANDLED_SIGNALS = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP) +PR_SET_PDEATHSIG = 1 + + +class ParentInterrupted(BaseException): + def __init__(self, signum: int) -> None: + self.signum = signum + + +def fail() -> None: + raise SystemExit("service-context runner failed closed") + + +def process_cgroup(pid: str) -> str: + try: + lines = Path(f"/proc/{pid}/cgroup").read_text(encoding="ascii").splitlines() + except (OSError, UnicodeError): + fail() + matches = [] + for line in lines: + parts = line.split(":", 2) + if len(parts) == 3 and parts[0] == "0" and parts[1] == "": + matches.append(parts[2]) + if len(matches) != 1: + fail() + value = matches[0] + path = PurePosixPath(value) + if not value.startswith("/") or ".." in path.parts or str(path) != value: + fail() + return value + + +def set_parent_death_signal(expected_parent: int) -> None: + libc = ctypes.CDLL(None, use_errno=True) + prctl = libc.prctl + prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] + prctl.restype = ctypes.c_int + if prctl(PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) != 0: + os._exit(125) + if os.getppid() != expected_parent: + os._exit(125) + + +def kill_and_reap(child: int) -> None: + try: + os.kill(child, signal.SIGKILL) + except ProcessLookupError: + pass + while True: + try: + os.waitpid(child, 0) + return + except InterruptedError: + continue + except ChildProcessError: + return + + +def supported_resource_ids() -> tuple[int, ...]: + identifiers = set() + for name in dir(resource): + if not name.startswith("RLIMIT_") or name == "RLIMIT_NLIMITS": + continue + identifier = getattr(resource, name) + if not isinstance(identifier, int) or identifier < 0: + continue + try: + resource.prlimit(os.getpid(), identifier) + except (OSError, ValueError): + continue + identifiers.add(identifier) + return tuple(sorted(identifiers)) + + +def copy_process_limits(source_pid: int, child: int) -> None: + snapshot = { + identifier: resource.prlimit(source_pid, identifier) + for identifier in supported_resource_ids() + } + for identifier, limits in snapshot.items(): + resource.prlimit(child, identifier, limits) + if resource.prlimit(child, identifier) != limits: + fail() + if any(resource.prlimit(source_pid, identifier) != limits for identifier, limits in snapshot.items()): + fail() + + +def no_op_stopped_child(_child: int) -> None: + return None + + +def run_stopped_child( + command: list[str], + expected_cgroup: str, + *, + move_child, + read_child_cgroup, + prepare_child=set_parent_death_signal, + configure_stopped_child=no_op_stopped_child, +) -> int: + if not command or not command[0].startswith("/"): + fail() + original_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + parent_pid = os.getpid() + try: + child = os.fork() + except BaseException: + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + raise + if child == 0: + try: + for signum in HANDLED_SIGNALS: + signal.signal(signum, signal.SIG_DFL) + prepare_child(parent_pid) + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + if os.getppid() != parent_pid: + os._exit(125) + os.kill(os.getpid(), signal.SIGSTOP) + os.execv(command[0], command) + except BaseException: + os._exit(126) + + previous_handlers = {} + reaped = False + interrupted = None + status = None + + def interrupt(signum, _frame) -> None: + signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + raise ParentInterrupted(signum) + + try: + for signum in HANDLED_SIGNALS: + previous_handlers[signum] = signal.signal(signum, interrupt) + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + waited, status = os.waitpid(child, os.WUNTRACED) + if waited != child or not os.WIFSTOPPED(status) or os.WSTOPSIG(status) != signal.SIGSTOP: + fail() + move_child(child) + if read_child_cgroup(str(child)) != expected_cgroup: + fail() + configure_stopped_child(child) + os.kill(child, signal.SIGCONT) + waited, status = os.waitpid(child, 0) + if waited != child: + fail() + reaped = True + except ParentInterrupted as exc: + interrupted = exc.signum + finally: + signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + if not reaped: + kill_and_reap(child) + for signum, previous in previous_handlers.items(): + signal.signal(signum, previous) + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + + if interrupted is not None: + signal.signal(interrupted, signal.SIG_DFL) + os.kill(os.getpid(), interrupted) + fail() + if status is None: + fail() + if os.WIFEXITED(status): + return os.WEXITSTATUS(status) + if os.WIFSIGNALED(status): + return 128 + os.WTERMSIG(status) + fail() + + +def main() -> int: + if len(sys.argv) < 4: + fail() + main_pid, expected_cgroup, *command = sys.argv[1:] + if not main_pid.isascii() or not main_pid.isdecimal() or int(main_pid) <= 0: + fail() + if process_cgroup(main_pid) != expected_cgroup: + fail() + cgroup_root = Path("/sys/fs/cgroup").resolve(strict=True) + cgroup_dir = (cgroup_root / expected_cgroup.lstrip("/")).resolve(strict=True) + try: + cgroup_dir.relative_to(cgroup_root) + except ValueError: + fail() + cgroup_procs = cgroup_dir / "cgroup.procs" + try: + if not stat.S_ISREG(cgroup_procs.stat().st_mode): + fail() + except OSError: + fail() + + def move_child(child: int) -> None: + with cgroup_procs.open("w", encoding="ascii") as handle: + handle.write(f"{child}\n") + + return run_stopped_child( + command, + expected_cgroup, + move_child=move_child, + read_child_cgroup=process_cgroup, + configure_stopped_child=lambda child: copy_process_limits(int(main_pid), child), + ) + + +if __name__ == "__main__": + raise SystemExit(main()) +PY +} + +assert_administrator_secret_denied() { + local main_pid probe_program + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + probe_program="$(cat <<'PY' +import json +import os +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request + +project, expected_email = sys.argv[1:] +metadata_headers = {"Metadata-Flavor": "Google"} +opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def metadata(path: str) -> bytes: + request = urllib.request.Request( + "http://169.254.169.254/computeMetadata/v1/" + path, + headers=metadata_headers, + ) + with opener.open(request, timeout=10) as response: + return response.read(65536) + + +try: + email = metadata("instance/service-accounts/default/email").decode("utf-8").strip() + if email != expected_email: + raise RuntimeError + token_payload = json.loads(metadata("instance/service-accounts/default/token")) + token = token_payload.get("access_token") + if token_payload.get("token_type") != "Bearer" or not isinstance(token, str) or not token: + raise RuntimeError + secret = urllib.parse.quote("gcp-teleo-pgvector-standby-postgres-password", safe="") + url = f"https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{secret}/versions/latest:access" + request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + try: + response = opener.open(request, timeout=10) + except urllib.error.HTTPError as error: + body = error.read(65536) + parsed = json.loads(body) + detail = parsed.get("error", {}) + message = str(detail.get("message", "")).lower() + if error.code != 403 or detail.get("status") != "PERMISSION_DENIED": + raise RuntimeError from None + if "secretmanager.versions.access" not in message: + raise RuntimeError + else: + response.close() + raise RuntimeError + gcloud_command = [ + "/usr/bin/gcloud", + "secrets", + "versions", + "access", + "latest", + "--secret=gcp-teleo-pgvector-standby-postgres-password", + f"--project={project}", + "--quiet", + ] + default_discovery_environment = os.environ.copy() + for key in ( + "CLOUDSDK_CONFIG", + "GOOGLE_APPLICATION_CREDENTIALS", + "PGPASSFILE", + "PGSERVICE", + "XDG_CONFIG_HOME", + ): + default_discovery_environment.pop(key, None) + default_discovery_environment["HOME"] = "/home/teleo" + for environment in (os.environ.copy(), default_discovery_environment): + denied = subprocess.run( + gcloud_command, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + env=environment, + timeout=30, + check=False, + ) + denial_error = denied.stderr[:65536] + if denied.returncode == 0: + raise RuntimeError + if "PERMISSION_DENIED" not in denial_error or "secretmanager.versions.access" not in denial_error: + raise RuntimeError +except Exception: + raise SystemExit("administrator secret was not the exact expected metadata-identity IAM denial") from None + +print(json.dumps({ + "administrator_secret_access": "iam_permission_denied", + "ambient_gcloud_administrator_secret_access": "iam_permission_denied", + "credential_context": "systemd_main_pid_environment", + "direct_metadata_administrator_secret_access": "iam_permission_denied", + "default_discovery_administrator_secret_access": "iam_permission_denied", + "identity_source": "gce_metadata_ambient_gcloud_and_default_discovery", + "stdout_discarded": True, +}, sort_keys=True)) +PY +)" + run_in_service_cgroup "$main_pid" \ + /usr/bin/nsenter -t "$main_pid" -m -n --env -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --clear-groups \ + --no-new-privs \ + /usr/bin/python3 -I -c "$probe_program" "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" +} + +run_scoped_status() { + local phase="$1" tool_path="$2" server_ca_path="${3:-$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem}" + local cloudsdk_config="${4:-$REMOTE_RUNTIME_BIN/gcloud-config}" + assert_attached_service_account + echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical" + sudo -n -u teleo env -i \ + HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + TELEO_GCP_PROJECT=teleo-501523 \ + TELEO_GCP_METADATA_ONLY=1 \ + TELEO_CLOUDSQL_HOST=10.61.0.3 \ + TELEO_CLOUDSQL_PORT=5432 \ + TELEO_CLOUDSQL_DB=teleo_canonical \ + TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical \ + TELEO_CLOUDSQL_USER=leoclean_kb_runtime \ + TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \ + TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \ + CLOUDSDK_CONFIG="$cloudsdk_config" \ + PGSSLMODE=verify-ca \ + PGSSLROOTCERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ + /usr/bin/python3 -I "$tool_path" status --format json --redacted +} + +run_service_wrapper_status() { + local phase="$1" main_pid + assert_attached_service_account + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + echo "SCOPED_WRAPPER_STATUS phase=$phase user=teleo database=teleo_canonical" + # --env imports the target MainPID's effective /proc environment. setpriv + # drops back to the service's Unix identity without sudo resetting that env. + run_in_service_cgroup "$main_pid" \ + /usr/bin/nsenter -t "$main_pid" -m -n --env -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --clear-groups \ + --no-new-privs \ + "$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted +} + +service_fingerprint() { + printf '%s|%s|%s|%s' \ + "$(systemctl show "$SERVICE" --property=ActiveState --value)" \ + "$(systemctl show "$SERVICE" --property=SubState --value)" \ + "$(systemctl show "$SERVICE" --property=MainPID --value)" \ + "$(systemctl show "$SERVICE" --property=NRestarts --value)" +} + +assert_service_fingerprint() { + local expected="$1" actual + actual="$(service_fingerprint)" + if [ "$actual" != "$expected" ]; then + echo "Service state changed during the read-only preflight." >&2 + return 1 + fi + echo "PREFLIGHT_SERVICE_UNCHANGED fingerprint=$actual" +} + +run_permission_verifier() { + local phase="$1" verifier_path="$2" receipt_path="$3" + local server_ca_path="${4:-$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem}" receipt_dir run_id receipt_state + local main_pid + local cloudsdk_config="${5:-$REMOTE_RUNTIME_BIN/gcloud-config}" + run_id="$(date -u +%Y%m%d%H%M%S)-$phase" + receipt_dir="$(dirname "$receipt_path")" + sudo -n -u teleo mkdir -p -- "$receipt_dir" + sudo -n -u teleo chmod 0700 "$receipt_dir" + echo "NEGATIVE_PERMISSION_RECEIPT phase=$phase run_id=$run_id" + if [ "$phase" = pre ]; then + sudo -n -u teleo env -i \ + HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ + /usr/bin/python3 -I "$verifier_path" --run-id "$run_id" --output "$receipt_path" + else + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + run_in_service_cgroup "$main_pid" \ + /usr/bin/nsenter -t "$main_pid" -m -n -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --clear-groups \ + --no-new-privs \ + /usr/bin/env -i \ + HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ + /usr/bin/python3 -I "$verifier_path" --run-id "$run_id" --output "$receipt_path" + fi + receipt_state="$(sudo stat -c '%U:%G:%a' "$receipt_path")" + if [ "$receipt_state" != "teleo:teleo:600" ]; then + echo "Permission receipt has unexpected ownership or mode: $receipt_state" >&2 + return 1 + fi + sudo /usr/bin/python3 -I - "$receipt_path" <<'PY' +import json +import sys +from pathlib import Path + +receipt = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +expected = { + "artifact": "gcp_leoclean_runtime_permissions", + "current_tier": "T3_live_readonly", + "mode": "live_private_gcp_staging", + "required_tier": "T3_live_readonly", + "status": "pass", +} +mismatched = sorted(key for key, value in expected.items() if receipt.get(key) != value) +if mismatched: + raise SystemExit("permission receipt contract mismatch: " + ",".join(mismatched)) +print(json.dumps({"permission_receipt": "validated", "validated_fields": sorted(expected)}, sort_keys=True)) +PY +} + +assert_trusted_executable() { + local requested="$1" resolved current owner + sudo test -x "$requested" + owner="$(sudo stat -c '%U' "$requested")" + if [ "$owner" != root ]; then + echo "Runtime executable path is not root-owned: $requested" >&2 + return 1 + fi + resolved="$(sudo readlink -f -- "$requested")" + sudo test -f "$resolved" + sudo test -x "$resolved" + current="$(dirname "$requested")" + while :; do + test "$(sudo stat -Lc '%U' "$current")" = root + sudo -n -u teleo test ! -w "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done + current="$resolved" + while :; do + test "$(sudo stat -Lc '%U' "$current")" = root + sudo -n -u teleo test ! -w "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done +} + +assert_installed_files() { + local wrapper tool permission_verifier service_env_verifier server_ca gcloud_config dropin revision + local executable_dir protected_dir protected_file actual_entries expected_entries + wrapper="$REMOTE_RUNTIME_BIN/teleo-kb" + tool="$REMOTE_RUNTIME_BIN/cloudsql_memory_tool.py" + permission_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" + service_env_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" + server_ca="$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem" + gcloud_config="$REMOTE_RUNTIME_BIN/gcloud-config" + dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME" + revision="$REMOTE_RUNTIME_BIN/.livingip-runtime-revision" + test "$(sudo sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA" + test "$(sudo sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA" + test "$(sudo sha256sum "$permission_verifier" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" + test "$(sudo sha256sum "$service_env_verifier" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" + test "$(sudo sha256sum "$server_ca" | awk '{print $1}')" = "$SERVER_CA_SHA" + test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA" + test "$(sudo stat -c '%U:%G:%a' "$REMOTE_RUNTIME_BIN")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$wrapper")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$tool")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$permission_verifier")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$service_env_verifier")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$server_ca")" = "root:root:644" + test "$(sudo stat -c '%U:%G:%a' "$gcloud_config")" = "root:root:555" + test "$(sudo stat -c '%U:%G:%a' "$revision")" = "root:root:644" + test "$(sudo stat -c '%U:%G:%a' "$dropin")" = "root:root:644" + test "$(sudo cat "$revision")" = "$SOURCE_REVISION" + sudo test -z "$(sudo find "$gcloud_config" -mindepth 1 -print -quit)" + expected_entries="$(printf '%s\n' .livingip-runtime-revision cloudsql-server-ca.pem cloudsql_memory_tool.py gcloud-config teleo-kb verify_gcp_leoclean_runtime_permissions.py verify_gcp_leoclean_service_environment.py | sort)" + actual_entries="$(sudo find "$REMOTE_RUNTIME_BIN" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" + if [ "$actual_entries" != "$expected_entries" ]; then + echo "Runtime directory contains an unreviewed or missing entry." >&2 + return 1 + fi + for protected_dir in \ + /usr \ + /usr/local \ + /usr/local/libexec \ + /usr/local/libexec/livingip \ + "$REMOTE_RUNTIME_BIN" \ + "$gcloud_config" \ + "$REMOTE_DROPIN_DIR"; do + sudo test -d "$protected_dir" + sudo test ! -L "$protected_dir" + test "$(sudo stat -c '%U:%G' "$protected_dir")" = "root:root" + sudo -n -u teleo test ! -w "$protected_dir" + done + for protected_file in \ + "$wrapper" \ + "$tool" \ + "$permission_verifier" \ + "$service_env_verifier" \ + "$server_ca" \ + "$revision" \ + "$dropin"; do + sudo test -f "$protected_file" + sudo test ! -L "$protected_file" + sudo -n -u teleo test ! -w "$protected_file" + done + for executable_dir in \ + /usr/local/sbin \ + /usr/local/bin \ + /usr/sbin \ + /usr/bin \ + /sbin \ + /bin; do + sudo test -d "$executable_dir" + test "$(sudo stat -Lc '%U:%G' "$executable_dir")" = "root:root" + sudo -n -u teleo test ! -w "$executable_dir" + done + for protected_file in /bin/bash /usr/bin/curl /usr/bin/env /usr/bin/gcloud /usr/bin/psql /usr/bin/python3 /usr/bin/setpriv; do + assert_trusted_executable "$protected_file" + done + sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin" + sudo grep -E '^Environment=TELEO_(KB_MODE|GCP_PROJECT|GCP_METADATA_ONLY|CLOUDSQL_|CANONICAL_CLOUDSQL_DB)' "$dropin" + sudo grep -Fx 'ReadOnlyPaths=/home/teleo' "$dropin" + sudo grep -Fx "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$dropin" + sudo grep -Fx "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$dropin" + sudo grep -Fx 'CapabilityBoundingSet=' "$dropin" + sudo grep -Fx 'AmbientCapabilities=' "$dropin" + sudo grep -Fx 'NoNewPrivileges=yes' "$dropin" + sudo grep -Fx 'Group=teleo' "$dropin" + sudo grep -Fx 'SupplementaryGroups=teleo' "$dropin" + sudo grep -Fx \ + 'UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT' \ + "$dropin" +} + +assert_deployed_files() { + local main_pid namespace_state source_state mount_state + local home_mount_state state_mount_state workspace_mount_state capability_set ambient_capabilities + assert_installed_files + + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + source_state="$(stat -Lc '%d:%i' "$REMOTE_RUNTIME_BIN")" + namespace_state="$(sudo nsenter -t "$main_pid" -m -- stat -Lc '%d:%i' "$SERVICE_PROFILE_BIN")" + if [ "$namespace_state" != "$source_state" ]; then + echo "Service namespace does not bind the reviewed runtime directory." >&2 + return 1 + fi + mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -T "$SERVICE_PROFILE_BIN" -n -o OPTIONS)" + case ",$mount_state," in + *,ro,*) ;; + *) + echo "Service runtime bind is not read-only." >&2 + return 1 + ;; + esac + home_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M /home/teleo -n -o OPTIONS)" + state_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/state" -n -o OPTIONS)" + workspace_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/workspace" -n -o OPTIONS)" + case ",$home_mount_state," in + *,ro,*) ;; + *) + echo "Service home mount is not read-only." >&2 + return 1 + ;; + esac + case ",$state_mount_state," in + *,rw,*) ;; + *) + echo "Service state mount is not writable." >&2 + return 1 + ;; + esac + case ",$workspace_mount_state," in + *,rw,*) ;; + *) + echo "Service workspace mount is not writable." >&2 + return 1 + ;; + esac + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w /home + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT/profiles" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_ROOT" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_BIN" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/state" + sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/workspace" + capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)" + ambient_capabilities="$(systemctl show "$SERVICE" --property=AmbientCapabilities --value)" + if [ -n "$capability_set" ] || [ -n "$ambient_capabilities" ]; then + echo "Service effective capability sets are not empty." >&2 + return 1 + fi + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb" | awk '{print $1}')" = "$WRAPPER_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql_memory_tool.py" | awk '{print $1}')" = "$TOOL_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_runtime_permissions.py" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_service_environment.py" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql-server-ca.pem" | awk '{print $1}')" = "$SERVER_CA_SHA" +} +REMOTE +) + +printf -v verify_context \ + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nEXPECTED_HERMES_EXECUTABLE=%q\nEXPECTED_HERMES_PYTHON=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nSERVER_CA_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$EXPECTED_HERMES_EXECUTABLE" "$EXPECTED_HERMES_PYTHON" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ + "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ + "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$SERVER_CA_SHA" "$DROPIN_SHA" "$SOURCE_REVISION" + +verify_body=$(cat <<'REMOTE' +set -euo pipefail +verification_tmp="$(sudo mktemp -d /run/teleo-gcp-runtime-verify.XXXXXX)" +sudo chown teleo:teleo "$verification_tmp" +sudo chmod 0700 "$verification_tmp" +cleanup_verification() { sudo rm -rf -- "$verification_tmp"; } +trap cleanup_verification EXIT +assert_deployed_files +assert_service_running +verification_service_before="$(service_fingerprint)" +assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" +run_service_wrapper_status verify +assert_administrator_secret_denied +run_permission_verifier verify "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" "$verification_tmp/permission-receipt.json" +assert_service_fingerprint "$verification_service_before" +assert_service_running +systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager +REMOTE +) +verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}" + +if $DRY_RUN; then + echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE preflight_only=$PREFLIGHT_ONLY restart=$RESTART" + echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA server_ca_sha256=$SERVER_CA_SHA dropin_sha256=$DROPIN_SHA" + exit 0 +fi + +if $VERIFY_ONLY; then + "${GCP_SSH[@]}" --command="$verify_command" + exit 0 +fi + +if ! $RESTART && ! $PREFLIGHT_ONLY; then + echo "Refusing to install runtime files without --restart; use --dry-run, --preflight-only, or --verify-only for read-only work." >&2 + exit 64 +fi + +printf -v sync_context \ + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nEXPECTED_HERMES_EXECUTABLE=%q\nEXPECTED_HERMES_PYTHON=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nSERVER_CA_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nSERVER_CA_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$EXPECTED_HERMES_EXECUTABLE" "$EXPECTED_HERMES_PYTHON" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ + "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ + "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$SERVER_CA_SHA" "$DROPIN_SHA" \ + "$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$SERVER_CA_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY" + +sync_body=$(cat <<'REMOTE' +set -euo pipefail +remote_tmp="$(sudo mktemp -d /run/teleo-gcp-leoclean-runtime.XXXXXX)" +runtime_bin="$REMOTE_RUNTIME_BIN" +runtime_parent="$(dirname "$runtime_bin")" +runtime_grandparent="$(dirname "$runtime_parent")" +runtime_anchor="$(dirname "$runtime_grandparent")" +dropin_dir="$REMOTE_DROPIN_DIR" +dropin_parent="$(dirname "$dropin_dir")" +payload_dir="$remote_tmp/payload" +backup_dir="$remote_tmp/backup" +work_dir="$remote_tmp/work" +wrapper_stage="$runtime_bin/.teleo-kb.livingip-new.$$" +tool_stage="$runtime_bin/.cloudsql-memory-tool.livingip-new.$$" +permission_verifier_stage="$runtime_bin/.runtime-permission-verifier.livingip-new.$$" +service_env_verifier_stage="$runtime_bin/.service-environment-verifier.livingip-new.$$" +server_ca_stage="$runtime_bin/.cloudsql-server-ca.livingip-new.$$" +revision_stage="$runtime_bin/.livingip-runtime-revision.livingip-new.$$" +dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$" +mutation_started=false + +assert_existing_root_directory() { + local target="$1" + if sudo test -e "$target" || sudo test -L "$target"; then + sudo test -d "$target" + sudo test ! -L "$target" + test "$(sudo stat -c '%U:%G' "$target")" = "root:root" + sudo -n -u teleo test ! -w "$target" + fi +} + +assert_existing_root_file() { + local target="$1" + if sudo test -e "$target" || sudo test -L "$target"; then + sudo test -f "$target" + sudo test ! -L "$target" + test "$(sudo stat -c '%U:%G' "$target")" = "root:root" + sudo -n -u teleo test ! -w "$target" + fi +} + +backup_path() { + local target="$1" name="$2" + if sudo test -e "$target"; then + sudo cp -a -- "$target" "$backup_dir/$name" + sudo touch "$backup_dir/$name.present" + fi +} + +backup_directory_state() { + local target="$1" name="$2" + if sudo test -d "$target"; then + sudo stat -c '%u %g %a' "$target" | sudo tee "$backup_dir/$name.metadata" >/dev/null + sudo touch "$backup_dir/$name.present" + fi +} + +restore_directory_state() { + local target="$1" name="$2" uid gid mode + if sudo test -f "$backup_dir/$name.present"; then + read -r uid gid mode <<<"$(sudo cat "$backup_dir/$name.metadata")" + sudo chown "$uid:$gid" "$target" + sudo chmod "$mode" "$target" + elif sudo test -d "$target"; then + sudo rmdir -- "$target" + fi +} + +restore_path() { + local target="$1" name="$2" restore_rc=0 restore_tmp + if sudo test -f "$backup_dir/$name.present"; then + restore_tmp="${target}.livingip-rollback.$$" + sudo rm -f -- "$restore_tmp" || restore_rc=1 + if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then + sudo mv -f -- "$restore_tmp" "$target" || restore_rc=1 + else + restore_rc=1 + fi + sudo rm -f -- "$restore_tmp" || restore_rc=1 + else + sudo rm -f -- "$target" || restore_rc=1 + fi + return "$restore_rc" +} + +validate_restored_path() { + local target="$1" name="$2" + if sudo test -f "$backup_dir/$name.present"; then + sudo test -f "$target" + sudo test ! -L "$target" + sudo cmp -s -- "$backup_dir/$name" "$target" + test "$(sudo stat -c '%u:%g:%a' "$target")" = \ + "$(sudo stat -c '%u:%g:%a' "$backup_dir/$name")" + else + sudo test ! -e "$target" + sudo test ! -L "$target" + fi +} + +validate_restored_directory() { + local target="$1" name="$2" expected_state + if sudo test -f "$backup_dir/$name.present"; then + expected_state="$(sudo cat "$backup_dir/$name.metadata" | tr ' ' ':')" + sudo test -d "$target" + sudo test ! -L "$target" + test "$(sudo stat -c '%u:%g:%a' "$target")" = "$expected_state" + else + sudo test ! -e "$target" + sudo test ! -L "$target" + fi +} + +rollback_runtime() { + local rollback_rc=0 + echo "Deployment failed; restoring the complete previous runtime set." >&2 + sudo systemctl stop "$SERVICE" || rollback_rc=1 + restore_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1 + restore_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 + restore_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 + restore_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 + restore_path "$runtime_bin/cloudsql-server-ca.pem" server_ca || rollback_rc=1 + restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 + restore_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 + restore_directory_state "$runtime_bin/gcloud-config" gcloud_config || rollback_rc=1 + restore_directory_state "$runtime_bin" runtime_bin || rollback_rc=1 + restore_directory_state "$runtime_parent" runtime_parent || rollback_rc=1 + restore_directory_state "$runtime_grandparent" runtime_grandparent || rollback_rc=1 + restore_directory_state "$dropin_dir" dropin_dir || rollback_rc=1 + if [ "$rollback_rc" -eq 0 ]; then + validate_restored_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1 + validate_restored_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 + validate_restored_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 + validate_restored_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 + validate_restored_path "$runtime_bin/cloudsql-server-ca.pem" server_ca || rollback_rc=1 + validate_restored_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 + validate_restored_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 + validate_restored_directory "$runtime_grandparent" runtime_grandparent || rollback_rc=1 + validate_restored_directory "$runtime_parent" runtime_parent || rollback_rc=1 + validate_restored_directory "$runtime_bin" runtime_bin || rollback_rc=1 + validate_restored_directory "$runtime_bin/gcloud-config" gcloud_config || rollback_rc=1 + validate_restored_directory "$dropin_dir" dropin_dir || rollback_rc=1 + fi + if [ "$rollback_rc" -eq 0 ]; then + sudo systemctl daemon-reload || rollback_rc=1 + fi + if [ "$rollback_rc" -eq 0 ]; then + assert_unit_configuration false || rollback_rc=1 + fi + if [ "$rollback_rc" -eq 0 ]; then + sudo systemctl start "$SERVICE" || rollback_rc=1 + assert_service_running false || rollback_rc=1 + fi + return "$rollback_rc" +} + +on_exit() { + local rc="$1" rollback_rc=0 + trap - EXIT INT TERM + if [ "$rc" -ne 0 ] && [ "$mutation_started" = true ]; then + set +e + rollback_runtime + rollback_rc=$? + set -e + if [ "$rollback_rc" -ne 0 ]; then + echo "CRITICAL: previous runtime restoration did not return the service to active/running." >&2 + rc=1 + fi + fi + sudo rm -f -- \ + "$wrapper_stage" \ + "$tool_stage" \ + "$permission_verifier_stage" \ + "$service_env_verifier_stage" \ + "$server_ca_stage" \ + "$revision_stage" \ + "$dropin_stage" || true + sudo rm -rf -- "$remote_tmp" || true + exit "$rc" +} + +trap 'on_exit $?' EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +sudo chmod 0711 "$remote_tmp" +sudo install -d -o root -g root -m 0755 "$payload_dir" +sudo install -d -o root -g root -m 0700 "$backup_dir" +sudo install -d -o teleo -g teleo -m 0700 "$work_dir" +sudo tar --no-same-owner --no-same-permissions --no-xattrs --no-acls --no-selinux -C "$payload_dir" -xf - +if sudo find "$payload_dir" -type l -print -quit | grep -q .; then + echo "Staged runtime payload contains a symlink." >&2 + exit 65 +fi +if sudo find "$payload_dir" ! -type d ! -type f -print -quit | grep -q .; then + echo "Staged runtime payload contains a non-regular entry." >&2 + exit 65 +fi +sudo chown -R root:root "$payload_dir" +sudo find "$payload_dir" -type d -exec chmod 0555 {} + +sudo find "$payload_dir" -type f -exec chmod 0444 {} + +sudo install -d -o root -g root -m 0555 "$payload_dir/gcloud-config" +sudo chmod 0555 \ + "$payload_dir/$WRAPPER_REL" \ + "$payload_dir/$TOOL_REL" \ + "$payload_dir/$PERMISSION_VERIFIER_REL" \ + "$payload_dir/$SERVICE_ENV_VERIFIER_REL" +test "$(sudo sha256sum "$payload_dir/$SERVER_CA_REL" | awk '{print $1}')" = "$SERVER_CA_SHA" +test "$(sudo sha256sum "$payload_dir/$WRAPPER_REL" | awk '{print $1}')" = "$WRAPPER_SHA" +test "$(sudo sha256sum "$payload_dir/$TOOL_REL" | awk '{print $1}')" = "$TOOL_SHA" +test "$(sudo sha256sum "$payload_dir/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" +test "$(sudo sha256sum "$payload_dir/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" +test "$(sudo sha256sum "$payload_dir/$DROPIN_REL" | awk '{print $1}')" = "$DROPIN_SHA" +test "$(sudo stat -c '%U:%G:%a' "$remote_tmp")" = "root:root:711" +test "$(sudo stat -c '%U:%G:%a' "$payload_dir")" = "root:root:555" +test "$(sudo stat -c '%U:%G:%a' "$backup_dir")" = "root:root:700" +test "$(sudo stat -c '%U:%G:%a' "$work_dir")" = "teleo:teleo:700" +sudo -n -u teleo test ! -w "$remote_tmp" +sudo -n -u teleo test ! -w "$payload_dir" +sudo -n -u teleo test ! -w "$backup_dir" +sudo test -d "$runtime_anchor" +sudo test -d "$dropin_parent" +sudo test ! -L "$runtime_anchor" +sudo test ! -L "$dropin_parent" +test "$(sudo stat -c '%U:%G' "$runtime_anchor")" = "root:root" +test "$(sudo stat -c '%U:%G' "$dropin_parent")" = "root:root" +sudo -n -u teleo test ! -w "$runtime_anchor" +sudo -n -u teleo test ! -w "$dropin_parent" +for existing_directory in \ + "$runtime_grandparent" \ + "$runtime_parent" \ + "$runtime_bin" \ + "$runtime_bin/gcloud-config" \ + "$dropin_dir"; do + assert_existing_root_directory "$existing_directory" +done +for existing_file in \ + "$runtime_bin/teleo-kb" \ + "$runtime_bin/cloudsql_memory_tool.py" \ + "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" \ + "$runtime_bin/verify_gcp_leoclean_service_environment.py" \ + "$runtime_bin/cloudsql-server-ca.pem" \ + "$runtime_bin/.livingip-runtime-revision" \ + "$dropin_dir/$DROPIN_NAME"; do + assert_existing_root_file "$existing_file" +done +if sudo test -d "$runtime_bin/gcloud-config"; then + sudo test -z "$(sudo find "$runtime_bin/gcloud-config" -mindepth 1 -print -quit)" +fi + +assert_service_running false +if [ "$PREFLIGHT_ONLY" = true ]; then + preflight_service_before="$(service_fingerprint)" + assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false + run_scoped_status pre "$payload_dir/$TOOL_REL" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" + assert_administrator_secret_denied + run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" + assert_service_fingerprint "$preflight_service_before" + assert_service_running false + systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager + exit 0 +fi + +backup_path "$runtime_bin/teleo-kb" wrapper +backup_path "$runtime_bin/cloudsql_memory_tool.py" tool +backup_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier +backup_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier +backup_path "$runtime_bin/cloudsql-server-ca.pem" server_ca +backup_path "$dropin_dir/$DROPIN_NAME" dropin +backup_path "$runtime_bin/.livingip-runtime-revision" revision +backup_directory_state "$runtime_parent" runtime_parent +backup_directory_state "$runtime_grandparent" runtime_grandparent +backup_directory_state "$runtime_bin" runtime_bin +backup_directory_state "$runtime_bin/gcloud-config" gcloud_config +backup_directory_state "$dropin_dir" dropin_dir + +preflight_service_before="$(service_fingerprint)" +assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false +run_scoped_status pre "$payload_dir/$TOOL_REL" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" +assert_administrator_secret_denied +run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" +assert_service_fingerprint "$preflight_service_before" +assert_service_running false +mutation_started=true +sudo systemctl stop "$SERVICE" +sudo install -d -o root -g root -m 0755 "$runtime_parent" +sudo install -d -o root -g root -m 0755 "$runtime_bin" +sudo install -d -o root -g root -m 0555 "$runtime_bin/gcloud-config" +sudo install -o root -g root -m 0644 "$payload_dir/$SERVER_CA_REL" "$server_ca_stage" +sudo mv -f -- "$server_ca_stage" "$runtime_bin/cloudsql-server-ca.pem" +sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage" +sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage" +sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" "$permission_verifier_stage" +sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" "$service_env_verifier_stage" +sudo mv -f -- "$wrapper_stage" "$runtime_bin/teleo-kb" +sudo mv -f -- "$tool_stage" "$runtime_bin/cloudsql_memory_tool.py" +sudo mv -f -- "$permission_verifier_stage" "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" +sudo mv -f -- "$service_env_verifier_stage" "$runtime_bin/verify_gcp_leoclean_service_environment.py" +sudo install -d -o root -g root -m 0755 "$dropin_dir" +sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL" "$dropin_stage" +sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME" +printf '%s\n' "$SOURCE_REVISION" | sudo tee "$revision_stage" >/dev/null +sudo chown root:root "$revision_stage" +sudo chmod 0644 "$revision_stage" +sudo mv -f -- "$revision_stage" "$runtime_bin/.livingip-runtime-revision" +assert_installed_files +sudo systemctl daemon-reload +assert_unit_configuration +sudo systemctl start "$SERVICE" + +assert_deployed_files +assert_service_running +post_service_before="$(service_fingerprint)" +assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py" +run_service_wrapper_status post +assert_administrator_secret_denied +run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" "$work_dir/permission-post/receipt.json" +assert_service_fingerprint "$post_service_before" +assert_service_running +systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager +mutation_started=false +REMOTE +) +sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}" + +COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \ + "$WRAPPER_REL" \ + "$TOOL_REL" \ + "$PERMISSION_VERIFIER_REL" \ + "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ + "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command" diff --git a/docs/gcp-leoclean-runtime-reconciliation.md b/docs/gcp-leoclean-runtime-reconciliation.md new file mode 100644 index 0000000..3ac69de --- /dev/null +++ b/docs/gcp-leoclean-runtime-reconciliation.md @@ -0,0 +1,320 @@ +# GCP Leo Runtime Reconciliation + +## Scope + +This runbook reconciles PR `#148`, the least-privilege follow-up to PR `#145`, +with the non-production GCP parallel Leo service and removes its dependency on +the PostgreSQL administrator credential. +It does not promote GCP, change the Telegram destination, copy newer VPS data, +or make `teleo_canonical` production-authoritative. + +Target surfaces: + +- project: `teleo-501523`; +- VM: `teleo-prod-1` in `europe-west6-a`; +- service: `leoclean-gcp-prod-parallel.service`; +- Cloud SQL database: `teleo_canonical` on private address `10.61.0.3`; +- runtime database role: `leoclean_kb_runtime`; +- staging-function owner: `leoclean_kb_stage_owner` (`NOLOGIN`); +- runtime secret: `gcp-teleo-pgvector-standby-leoclean-kb-runtime-password`. + +Observed before reconciliation on 2026-07-14: the service was +`active/running`, but `/usr/local/bin/teleo-kb` matched unmerged PR `#145`, the +effective systemd environment selected database user `postgres` and the +administrator password secret, and a live `teleo-kb status` call exceeded a +20-second bound. Secret Manager access and private PostgreSQL reachability both +passed independently, isolating the timeout to the old helper path rather than +IAM or networking. + +## Required End State + +1. The live wrapper and Cloud SQL helper hashes match a merged repository + commit. +2. `TELEO_KB_MODE=cloudsql`; missing tools or credentials fail closed. +3. Canonical zero-hit searches do not consult `teleo_restore` unless an + operator explicitly opts in. The service is pinned to database + `teleo_canonical` with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`. +4. Leo can read the named canonical tables and stage only a + `pending_review` proposal through `kb_stage.stage_leoclean_proposal(...)`. +5. Leo cannot directly insert, update, or delete `public.*` or + `kb_stage.kb_proposals`, forge the proposer identity, `SET ROLE` into a + broader principal, or execute reviewer/apply functions. +6. The VM runtime identity can access only the scoped password secret needed by + this path, not the PostgreSQL administrator password secret. +7. Deployment proves the attached VM service account from the metadata server, + uses an empty phase-local Cloud SDK configuration, and rolls the complete + prior runtime set back if installation or post-restart verification fails. +8. Post-restart status, permission, and administrator-secret denial probes join + both the actual service mount and network namespaces. Effective systemd + network, address-family, socket-bind, syscall-filter, and LSM properties must + match the reviewed unrestricted transport contract so a host-context helper + cannot produce a false runtime receipt. + +## Safe Order Of Operations + +### 1. Read-only access and IAM audit + +Confirm the operator can read the project, use IAP/OS Login for the VM, inspect +Cloud SQL metadata, and inspect Secret Manager IAM. Identify the VM service +account and determine whether its secret access is project-wide or +secret-specific. Do not remove a project-wide binding until every legitimately +required runtime secret has an equivalent secret-level binding. + +### 2. Merge reviewed repository code + +Run CI on the completed PR `#148` repair before deployment. The deployment +source must be the resulting merged commit, not a working tree or unmerged +branch. + +### 3. Create the scoped secret and grant only the VM runtime identity + +Create the scoped secret without printing its value. Add one randomly generated +version, then grant `roles/secretmanager.secretAccessor` on that secret to the +VM runtime service account. Do not put `PGPASSWORD` in systemd, a repository, +an artifact, or a command transcript. + +### 4. Provision the scoped PostgreSQL role once + +Run `ops/provision_gcp_leoclean_runtime_role.sh` as the Cloud SQL administrator +from the private VM path. Supply the new runtime password through +`TELEO_LEOCLEAN_DB_PASSWORD` and the administrator password through +`PGPASSWORD`. The wrapper captures and unsets both fields before its first +external child, pins the private endpoint and reviewed server CA, detaches +`psql` from the controlling terminal, and feeds the runtime value only to +psql's client-side `\password` prompt. The SQL file and argv never contain the +cleartext runtime password. The migration: + +- creates or rotates `leoclean_kb_runtime`; +- changes any pre-existing `leoclean_kb_runtime` to `NOLOGIN`, removes its + membership edges, and terminates its existing sessions before password + handling; an interruption therefore leaves the old login disabled; +- enables `LOGIN` only as the last statement in the same transaction as the + database ACL, canonical grant, routine, large-object, and topology checks; +- creates a dedicated `NOLOGIN` function owner with no role memberships; +- removes stale table, column, sequence, function, and role-membership grants; +- inventories existing principals, replaces `PUBLIC CONNECT` with explicit + preserved grants, gives the runtime `CONNECT` only to `teleo_canonical`, and + leaves the staging owner with no database connection target; +- inventories and preserves existing non-scoped application-routine and + large-object access while removing `PUBLIC` and both scoped roles from all + other application routines and every persistent large-object mutator; +- grants exact canonical reads; +- creates a locked security-definer staging function that hard-codes both + `pending_review` and canonical proposer `leo`; +- grants no direct table writes; +- aborts if either scoped role owns or can reach anything outside the explicit + allowlist. + +PostgreSQL grants `TEMP` to `PUBLIC` by default. The migration removes any +direct scoped `TEMP` grant and reports the remaining effective privilege, but +does not revoke `TEMP` from `PUBLIC`: PostgreSQL has no per-role deny, so that +would be a database-wide behavior change requiring a separate inventory of +`kb_apply`, reviewer, and operator use. The staging function remains protected +by a `pg_catalog, pg_temp` search path, schema-qualified relations, a fixed +`session_user`, and a tested temporary-object shadowing denial. + +The administrator password is used only for this bounded bootstrap. Retain no +password output. + +`teleo_kb`, `template1`, and every other noncanonical database are actual +negative connection checks. PostgreSQL's startup protocol does not expose a +SQLSTATE through `psql`, so the verifier requires the exact C-locale +`permission denied for database` and `does not have CONNECT privilege` +diagnostics in addition to the catalog proof that the runtime's sole effective +target is `teleo_canonical`. Audit fallback remains disabled with +`TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`. + +### 5. Preflight and deploy the merged runtime + +From the exact merged checkout, run: + +```bash +deploy/sync-gcp-leoclean-runtime.sh --dry-run +deploy/sync-gcp-leoclean-runtime.sh --preflight-only +deploy/sync-gcp-leoclean-runtime.sh --restart +``` + +`--preflight-only` extracts the candidate helper and permission verifier into a +root-created, root-owned, non-writable VM payload directory, verifies the actual +service `MainPID` environment, runs the candidate private Cloud SQL status path, +and runs the sanitized positive-and-negative permission verifier. A separate +`teleo`-owned directory holds only sanitized receipts; it never feeds an install +or rollback. The preflight does not stop or restart the service and does not +install or replace runtime files. It +captures `ActiveState`, `SubState`, `MainPID`, and `NRestarts` before and after +the checks and fails if any value changes. The preflight must pass before +`--restart` is attempted. + +The deploy script refuses dirty, symlinked, or unmerged runtime files and +refuses a live install without `--restart`. Its payload and rollback backup are +root-owned and non-writable by `teleo`; existing runtime intermediates and +targets must be real root-owned, non-writable paths. After preflight it stops +the parallel service, atomically replaces the existing Cloud SQL systemd drop-in, installs the +reviewed wrapper, helper, and permission verifier into the root-controlled +`/usr/local/libexec/livingip/leoclean-kb` directory, records the Git revision, +and verifies every installed hash, owner, mode, path type, and revision before +systemd reloads the unit or starts the service. The wrapper uses fixed +`/bin/bash` and `/usr/bin/python3` interpreters, and every directory on the +service `PATH` must be root-owned and non-writable by `teleo`. Every ancestor of that +directory, the directory itself, the wrapper, helper, verifier, and revision +marker must be non-writable by `teleo`. The systemd service receives that +directory as a read-only bind at its existing +`/home/teleo/.hermes/profiles/leoclean/bin` path; verification compares the +source and service-namespace inode and hashes and proves the mount is read-only. +To prevent the `teleo` service user from renaming a writable ancestor and +recreating that absolute path, the service namespace mounts the complete +`/home/teleo` tree read-only, allows writes back only to the profile's `state` +and `workspace` subdirectories, clears every bounding and ambient capability, +and enforces `NoNewPrivileges`. Live verification requires the service home and +bound `bin` to be read-only, both allowed subdirectories to be writable, the +exact `teleo` group set, and every process capability field to be zero. + +Preflight, post-restart, and `--verify-only` checks inspect the environment of the actual +systemd `MainPID` through `/proc//environ`, rather than trusting the +configured unit environment alone. It also rejects effective credential +properties, auxiliary `Exec*` hooks, any `EnvironmentFile`, and any drop-in +ordered after the reviewed configuration. A standalone fail-closed verifier requires +the exact reviewed private target and runtime role, requires +`TELEO_GCP_METADATA_ONLY=1`, and rejects all PostgreSQL, Cloud SDK, proxy, TLS +trust, broader Google credential, shell-startup, dynamic-loader, Python-loader, +administrator-secret, excerpt, credential-mode, claim-base, and retry-count +overrides without printing their values. The reviewed root-owned empty +`CLOUDSDK_CONFIG`, pinned `PGSSLMODE=verify-ca`, +server CA, and Python isolation fields must match exactly. This blocks +pre-wrapper injection through fields such as `BASH_ENV`, any `LD_*`, +`PYTHONPATH`, `PYTHONHTTPSVERIFY`, or `SSLKEYLOGFILE`. The checks also require +the reviewed drop-in to appear in `DropInPaths`. The wrapper status probe then +imports the validated environment from the actual `MainPID` with `nsenter +--env`, enters its mount and network namespaces, drops to `teleo`, and invokes the bound +wrapper without reinjecting any database identity setting. + +The effective unit must use the host network namespace (`PrivateNetwork=no`), +have no `NetworkNamespacePath`, `IPAddressDeny`, `IPAddressAllow`, +`RestrictAddressFamilies`, `SocketBindAllow`, `SocketBindDeny`, +`RestrictNetworkInterfaces`, `SystemCallFilter`, `AppArmorProfile`, or +`SELinuxContext` override, and load no +later drop-in. The executable regression harness injects each conflicting +property and requires the gate to fail. These checks cover restrictions that a +new `nsenter` process would not inherit merely by sharing the service network +namespace. + +For cgroup/eBPF restrictions, post-restart probes do not rely on enumeration. +A root-only bounded runner verifies the service's single cgroup-v2 membership +against systemd's effective `ControlGroup`, forks a stopped child, moves only +that child into the service cgroup, confirms the move, and then resumes it to +join the service namespaces and drop to `teleo`. It propagates exit or signal +status and reaps the child on every path. The child installs a Linux +parent-death `SIGKILL` before stopping, with a parent-PID race check; the parent +blocks and handles `INT`, `TERM`, and `HUP` through cleanup before re-raising +the signal. Before resume, every supported Linux `RLIMIT_*` soft/hard pair is +copied from the live `MainPID` with a stable source and child readback. Live +`MainPID` verification also requires seccomp mode and filter count zero, the +exact `unconfined` LSM context, `PrivateUsers=no`, and the same user-namespace +inode as PID 1, preventing a per-process filter, profile, resource ceiling, or +user namespace from being mistaken for the reviewed service context. + +The service runtime bypasses Cloud SDK configuration and credential caches +entirely. The helper obtains the attached service-account email and a bounded +access token from the fixed GCE metadata endpoint, requires the exact expected +service account, and accesses only the named scoped Secret Manager version over +the fixed HTTPS API. Its URL opener disables proxies. The scoped status read +and administrator-secret IAM denial are rerun after restart. The denial probe +imports the actual systemd `MainPID` environment and mount and network +namespaces, drops to +the service's `teleo` identity, and must observe the exact +`secretmanager.versions.access` permission denial. No token or +secret value is retained. The status read and permission verifier execute from +inside the service mount and network namespaces; the status path invokes the +bound `teleo-kb` wrapper and its adjacent helper rather than bypassing them with +a direct helper invocation. A failure +after mutation restores the prior runtime/drop-in directory existence and +metadata, wrapper, helper, both verifiers, drop-in, and revision as one set, +validates it against the root-only backup, and only then reloads systemd and +restarts the old service. + +### 6. Verify positive and negative behavior + +`--preflight-only`, `--restart`, and `--verify-only` run the verifier as the +`teleo` Unix user. To rerun the installed verifier directly on the VM and +retain its sanitized JSON receipt, use: + +```bash +receipt_dir="$(mktemp -d /tmp/leoclean-permission-receipt.XXXXXX)" +sudo chown teleo:teleo "$receipt_dir" +sudo chmod 0700 "$receipt_dir" +run_id="$(date -u +%Y%m%d%H%M%S)-manual" +sudo -n -u teleo env -i HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + /usr/bin/python3 -I /usr/local/libexec/livingip/leoclean-kb/verify_gcp_leoclean_runtime_permissions.py \ + --run-id "$run_id" \ + --output "$receipt_dir/permission-receipt.json" +sudo stat -c '%U:%G:%a' "$receipt_dir/permission-receipt.json" +``` + +The command must exit zero, print the same canonical JSON that it writes, and +leave the receipt as `teleo:teleo:600`. A passing receipt must report +`status=pass`, `mode=live_private_gcp_staging`, and both `required_tier` and +`current_tier` as `T3_live_readonly`. Those values are a receipt contract, not +a claim that this runbook has already been exercised successfully on the live +staging VM. + +Retain a redacted receipt containing: + +- merged Git commit and deployed file hashes; +- service `ActiveState`, `SubState`, `MainPID`, and `NRestarts`; +- metadata-server identity and an access token obtained with an empty Cloud SDK + configuration; +- `current_database()` and `current_user` showing + `teleo_canonical|leoclean_kb_runtime`; +- private server address `10.61.0.3`, port `5432`, and an active SSL session; +- exact safe runtime-role attributes and zero membership edges in + `pg_auth_members`; +- zero effective database/schema `CREATE`, persistent ownership, table/column + DML, sequence privileges, SELECT outside the exact table allowlist, or + executable unexpected `SECURITY DEFINER` functions, plus the exact staging + function owner/search-path/ACL contract; +- a real canonical status/search receipt; +- a transaction-rolled-back call to `stage_leoclean_proposal(...)` that returns + `pending_review` for canonical proposer `leo`, with zero matching canary rows + both before and after the transaction; +- denied direct insert, update, and delete on `kb_stage.kb_proposals`; +- denied insert, update, and delete on canonical `public.claims`; +- denied `lo_creat`, `lo_create`, and `lo_from_bytea`, zero effective + large-object mutator execution, and zero scoped large-object ownership/ACLs; +- denied reviewer/apply security-definer functions; +- exact function-catalog posture: only the five-argument staging function is + executable, the forged six-argument overload is absent, and each expected + reviewer/apply function exists but is not executable by the runtime role; +- unavailable forged-proposer overload and denied `SET ROLE` escalation into + the staging owner, reviewer, apply, or administrator roles; +- denied startup connections to both `teleo_kb` and `template1`, with the + catalog showing `teleo_canonical` as the sole runtime connection target and + zero remaining `PUBLIC CONNECT` grants; +- readable scoped runtime secret and an IAM permission denial for the + administrator secret, with neither secret value retained; +- zero Telegram messages and zero committed canary rows. + +### 7. Remove administrator-secret access + +Only after the scoped service passes post-restart verification, remove the VM +runtime identity's access path to +`gcp-teleo-pgvector-standby-postgres-password`. Verify from the VM runtime +identity that the scoped secret is readable and the administrator secret is +denied. Never delete the administrator secret merely to enforce runtime least +privilege; backup/restore operators may still require it under a separate +identity. + +## Stop Conditions + +Stop without cutover if any of these are true: + +- the deployed revision is not merged; +- the scoped status preflight fails; +- the role has direct proposal-table insert or canonical write permission; +- any approval/apply function is executable by the runtime role; +- removing broad Secret Manager access would break another required secret; +- GCP canonical rows are still stale relative to the chosen authority. + +The last condition does not invalidate this runtime security repair. It means +GCP remains staging and data reconciliation stays a separate, explicitly +authorized slice. diff --git a/docs/reports/leo-working-state-20260709/gcp-db-first-live-deploy-restart-current.json b/docs/reports/leo-working-state-20260709/gcp-db-first-live-deploy-restart-current.json index 651db21..9ad235f 100644 --- a/docs/reports/leo-working-state-20260709/gcp-db-first-live-deploy-restart-current.json +++ b/docs/reports/leo-working-state-20260709/gcp-db-first-live-deploy-restart-current.json @@ -6,6 +6,7 @@ "database_first_merge_commit": "08371bf2a0461536c112e1235a7229b52ed270cc", "database_first_pull_request": 144, "fail_closed_wrapper_commit": "8451c51", + "fail_closed_wrapper_merge_state_at_capture": "unmerged PR #145; manually installed on the GCP VM", "repository": "living-ip/teleo-infrastructure" }, "deployment": { @@ -55,7 +56,7 @@ "integration_regression": { "detected_live": true, "symptom": "The old auto-mode wrapper used a full receipted status call as a 30-second health probe; the stronger read exceeded that bound and the wrapper fell through to a different local tool.", - "repair": "Supported GCP knowledge commands now route directly to Cloud SQL and fail closed on errors instead of changing databases.", + "repair": "The manually installed PR #145 wrapper routes supported GCP knowledge commands directly to Cloud SQL and propagates failures after selection. Missing prerequisites could still fall through before selection at capture time.", "regression_tests": 2, "full_repository_tests": { "passed": 1240, @@ -113,8 +114,10 @@ "post_restart_model_turn_run": false }, "claim_ceiling": { - "proven": "Merged database-first helper and skill deployment, fail-closed Cloud SQL routing, controlled GCP service restart, and post-restart live status/search receipts.", + "proven": "Merged database-first helper and skill deployment, manually installed unmerged wrapper routing repair, controlled GCP service restart, and post-restart live status/search receipts.", "not_proven": [ + "repository/runtime parity for the wrapper", + "least-privilege Cloud SQL runtime credential", "post-restart model reasoning turn", "Telegram-visible delivery", "canonical proposal apply", diff --git a/docs/reports/leo-working-state-20260709/gcp-db-first-working-leo-20260714.md b/docs/reports/leo-working-state-20260709/gcp-db-first-working-leo-20260714.md index b33ff7b..d906df5 100644 --- a/docs/reports/leo-working-state-20260709/gcp-db-first-working-leo-20260714.md +++ b/docs/reports/leo-working-state-20260709/gcp-db-first-working-leo-20260714.md @@ -94,10 +94,11 @@ remaining target databases. The disabled rollback database still has zero connections. The live GCP Leo service remained PID `148735`, active/running, with `NRestarts=0` throughout restore, model replay, and cleanup. -### 5. The merged GCP path survived deployment and restart +### 5. The merged database-first path and an unmerged wrapper repair survived restart PR `#144` merged the database-first helper and skill. Their exact reviewed -hashes were installed on `teleo-prod-1`, then +hashes were installed on `teleo-prod-1`. A separate wrapper repair from the +then-unmerged PR `#145` was also installed manually, then `leoclean-gcp-prod-parallel.service` was intentionally restarted. It returned active/running with a new PID (`304036`) and the deployed hashes remained in place. @@ -105,9 +106,12 @@ place. The first post-restart search caught one real integration regression: the old wrapper used a full `status` read as a 30-second health probe. Stronger receipts made that probe exceed its bound, after which `auto` mode could silently select -a different local tool. The wrapper now routes every supported GCP knowledge -command directly to Cloud SQL and fails closed on errors. Two regression tests -cover direct routing and no fallback. +a different local tool. The manually deployed wrapper routes every supported +GCP knowledge command directly to Cloud SQL and propagates errors after backend +selection. At the time of this receipt, missing host prerequisites could still +cause an earlier fallback, the runtime still used the administrator database +credential, and the live wrapper did not match merged `main`. PR `#145` +therefore remained a reconciliation task rather than a production-ready result. A corrected live search then found the expected sandbagging claim plus both expected source rows and hashes from persistent `teleo_canonical`. A subsequent @@ -140,7 +144,7 @@ rows as if they were source-code branches. | `gcp-db-first-parity-current.json` | `0173ee6707016e8412e6dd4326d61f71b6ef862bbc5b819079d62680676729f2` | Schema, row, role, extension, and performance parity passed | | `gcp-db-first-blind-claim-current.json` | `8a7cc3c1814eb385e858f12ef7579cd4524f37886c03fc6d9c61624b6aff2a52` | Real GCP Hermes reasoning passed with source-bound read receipts | | `gcp-db-first-cleanup-current.json` | `cac7e34f45653fabb696d911b6ccc921d3f291bf8cbe05a429d757e717dcafb4` | Clone absent, rollback disabled, service unchanged | -| `gcp-db-first-live-deploy-restart-current.json` | `d70fdbac859b8c98d557f4df900139a93e6009e12f7b80c9adbd69a5655b9df7` | Merged deploy, fail-closed routing, restart, and live status/search readback passed | +| `gcp-db-first-live-deploy-restart-current.json` | `78e9dde38d641bfa3d58b2b7d8605b37f2861605707eddc0c94ad5c9a70d080d` | Merged PR `#144` files plus an unmerged PR `#145` wrapper were installed; restart and live status/search readback passed | ## What This Unlocks @@ -166,6 +170,9 @@ rows as if they were source-code branches. 6. A model reasoning turn after the controlled GCP restart. The post-restart proof in this report is the real service plus live `teleo-kb` status/search path, not a new paid model response. +7. Repository/runtime parity for the wrapper or a least-privilege Cloud SQL + credential. The captured service still used an unmerged wrapper and the + administrator database secret. The next product-level proof is one natural Telegram challenge that reaches the same database-grounded reasoning, followed by one explicitly approved diff --git a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py index a7fb4fe..2f964b0 100755 --- a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py +++ b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py @@ -1,20 +1,26 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 -I """Cloud SQL memory and KB-proposal bridge for the GCP leoclean profile. -Canonical claim/strategy application remains review-gated. Read commands query -the canonical Teleo KB first, then fall back to the restored `teleo_restore` -shadow schema in Cloud SQL. Proposal commands write staged review records into -`kb_stage.kb_proposals`; they do not apply canonical truth directly. +Canonical claim/strategy application remains review-gated. The production-like +Leo runtime is pinned to the private canonical Cloud SQL database and never +falls back to the restored `teleo_restore` shadow schema. Proposal commands +write staged review records into `kb_stage.kb_proposals`; they do not apply +canonical truth directly. """ from __future__ import annotations import argparse +import base64 +import binascii import hashlib import json import os import re +import stat import subprocess +import urllib.request +from pathlib import Path from typing import Any STOPWORDS = { @@ -74,6 +80,96 @@ STOPWORDS = { DEFAULT_CLAIM_BASE_URL = "https://leo.livingip.xyz" RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1" +RUNTIME_DB_USER = "leoclean_kb_runtime" +RUNTIME_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" +RUNTIME_GCP_PROJECT = "teleo-501523" +RUNTIME_CLOUDSQL_HOST = "10.61.0.3" +RUNTIME_CLOUDSQL_PORT = "5432" +RUNTIME_CLOUDSQL_DB = "teleo_canonical" +RUNTIME_SYSTEM_IDENTIFIER = "7659718422914359312" +RUNTIME_CLOUDSDK_CONFIG = "/usr/local/libexec/livingip/leoclean-kb/gcloud-config" +RUNTIME_PSQL_BIN = "/usr/bin/psql" +RUNTIME_SSL_MODE = "verify-ca" +RUNTIME_SSL_ROOT_CERT = "/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem" +RUNTIME_SSL_ROOT_CERT_SHA256 = "80701e768f0e1f6b9d621aa0b53f6e851daaa276c6d9a8e51a300fbc015539cb" +RUNTIME_SERVICE_ACCOUNT = "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" +GCE_METADATA_EMAIL_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" +GCE_METADATA_TOKEN_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" +RUNTIME_SECRET_ACCESS_URL = ( + "https://secretmanager.googleapis.com/v1/projects/teleo-501523/secrets/" + "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password/versions/latest:access" +) +METADATA_HEADERS = {"Metadata-Flavor": "Google"} +HTTP_TIMEOUT_SECONDS = 3.0 +MAX_METADATA_EMAIL_BYTES = 256 +MAX_METADATA_TOKEN_BYTES = 4096 +MAX_SECRET_RESPONSE_BYTES = 8192 +MAX_RUNTIME_SECRET_BYTES = 4096 +RUNTIME_TLS_TRUST_ENVIRONMENT = frozenset( + { + "CURL_CA_BUNDLE", + "PYTHONHTTPSVERIFY", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SSLKEYLOGFILE", + "OPENSSL_CONF", + "OPENSSL_ENGINES", + "OPENSSL_MODULES", + "GCONV_PATH", + } +) +RUNTIME_PSQL_ENV_BASE = { + "PATH": "/usr/bin:/bin", + "PGSSLMODE": RUNTIME_SSL_MODE, + "PGCONNECT_TIMEOUT": "8", +} + + +def runtime_ssl_root_cert() -> str: + """Return the reviewed installed or ephemeral preflight CA path.""" + + raw = os.environ.get("TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT", RUNTIME_SSL_ROOT_CERT) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + if resolved != candidate or not resolved.is_file(): + raise OSError + for part in (resolved, *resolved.parents): + metadata = part.stat() + if metadata.st_uid != 0 or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise OSError + certificate = resolved.read_bytes() + except OSError: + raise SystemExit("Reviewed Cloud SQL server CA path is unavailable or untrusted.") from None + if hashlib.sha256(certificate).hexdigest() != RUNTIME_SSL_ROOT_CERT_SHA256: + raise SystemExit("Reviewed Cloud SQL server CA does not match the pinned certificate.") + return str(resolved) + + +def runtime_cloudsdk_config() -> str: + """Return a root-owned empty SDK configuration directory.""" + + raw = os.environ.get("TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG", RUNTIME_CLOUDSDK_CONFIG) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + if resolved != candidate or not resolved.is_dir() or any(resolved.iterdir()): + raise OSError + for part in (resolved, *resolved.parents): + metadata = part.stat() + if metadata.st_uid != 0 or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise OSError + except OSError: + raise SystemExit("Reviewed empty Cloud SDK configuration is unavailable or untrusted.") from None + return str(resolved) + + +CLONE_READ_ONLY_PGOPTIONS = "-c default_transaction_read_only=on" RECEIPTED_READ_COMMANDS = frozenset( { "search", @@ -90,18 +186,36 @@ RECEIPTED_READ_COMMANDS = frozenset( ) +class _RejectRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *_args: Any, **_kwargs: Any) -> None: + return None + + +RUNTIME_PROXY_HANDLER = urllib.request.ProxyHandler({}) +RUNTIME_HTTP_OPENER = urllib.request.build_opener( + RUNTIME_PROXY_HANDLER, + _RejectRedirectHandler(), +) + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--host", default=os.environ.get("TELEO_CLOUDSQL_HOST", "10.61.0.3")) parser.add_argument("--port", default=os.environ.get("TELEO_CLOUDSQL_PORT", "5432")) - parser.add_argument("--db", default=os.environ.get("TELEO_CLOUDSQL_DB", "teleo_kb")) + parser.add_argument("--db", default=os.environ.get("TELEO_CLOUDSQL_DB", "teleo_canonical")) parser.add_argument("--canonical-db", default=os.environ.get("TELEO_CANONICAL_CLOUDSQL_DB", "teleo_canonical")) - parser.add_argument("--user", default=os.environ.get("TELEO_CLOUDSQL_USER", "postgres")) + parser.add_argument("--user", default=os.environ.get("TELEO_CLOUDSQL_USER")) parser.add_argument( "--password-secret", - default=os.environ.get("TELEO_CLOUDSQL_PASSWORD_SECRET", "gcp-teleo-pgvector-standby-postgres-password"), + default=os.environ.get("TELEO_CLOUDSQL_PASSWORD_SECRET"), ) parser.add_argument("--project", default=os.environ.get("TELEO_GCP_PROJECT", "teleo-501523")) + parser.add_argument( + "--credential-mode", + choices=["runtime", "clone-readonly"], + default=os.environ.get("TELEO_CLOUDSQL_CREDENTIAL_MODE", "runtime"), + help="Runtime mode requires the exact scoped Leo identity. Clone mode is limited to read-only teleo_clone_* tests.", + ) parser.add_argument("--format", choices=["markdown", "json"], default="markdown") parser.add_argument( "--include-excerpts", @@ -185,7 +299,7 @@ def parse_args() -> argparse.Namespace: propose.add_argument("--proposed", required=True, help="Proposed replacement or correction.") propose.add_argument("--evidence", action="append", default=[], help="Evidence/source pointer. Can be repeated.") propose.add_argument("--implication", action="append", default=[], help="Downstream implication. Can be repeated.") - propose.add_argument("--proposed-by", default="leo") + propose.add_argument("--proposed-by", choices=["leo"], default="leo") propose.add_argument("--originator", default="", help="Human or agent who originated the correction.") propose.add_argument("--channel", default="telegram") propose.add_argument("--source-ref", required=True) @@ -196,7 +310,7 @@ def parse_args() -> argparse.Namespace: propose_edge.add_argument("from_claim") propose_edge.add_argument("edge_type") propose_edge.add_argument("to_claim") - propose_edge.add_argument("--proposed-by", default="leo") + propose_edge.add_argument("--proposed-by", choices=["leo"], default="leo") propose_edge.add_argument("--channel", default="telegram") propose_edge.add_argument("--source-ref", required=True) propose_edge.add_argument("--rationale", required=True) @@ -228,6 +342,93 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() +def validate_runtime_connection(args: argparse.Namespace) -> None: + user = str(args.user or "") + password_secret = str(args.password_secret or "") + mode = str(getattr(args, "credential_mode", "runtime")) + if mode == "runtime": + ssl_root_cert = runtime_ssl_root_cert() + cloudsdk_config = runtime_cloudsdk_config() + if user != RUNTIME_DB_USER: + raise SystemExit(f"Leo runtime requires database user {RUNTIME_DB_USER}.") + if password_secret != RUNTIME_PASSWORD_SECRET: + raise SystemExit(f"Leo runtime requires scoped password secret {RUNTIME_PASSWORD_SECRET}.") + expected_fields = { + "project": RUNTIME_GCP_PROJECT, + "host": RUNTIME_CLOUDSQL_HOST, + "port": RUNTIME_CLOUDSQL_PORT, + "db": RUNTIME_CLOUDSQL_DB, + "canonical_db": RUNTIME_CLOUDSQL_DB, + } + for field, expected in expected_fields.items(): + actual = str(getattr(args, field, "") or "") + if actual != expected: + raise SystemExit(f"Leo runtime requires {field}={expected}.") + if os.environ.get("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") != "0": + raise SystemExit("Leo runtime requires TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0.") + if os.environ.get("TELEO_GCP_METADATA_ONLY") != "1": + raise SystemExit("Leo runtime requires TELEO_GCP_METADATA_ONLY=1.") + forbidden_environment = any( + ( + normalized_key.startswith("PG") + and not ( + (normalized_key == "PGSSLMODE" and value == RUNTIME_SSL_MODE) + or (normalized_key == "PGSSLROOTCERT" and value == ssl_root_cert) + ) + ) + or ( + normalized_key.startswith("CLOUDSDK_") + and not (normalized_key == "CLOUDSDK_CONFIG" and value == cloudsdk_config) + ) + or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS" + or normalized_key.startswith("LD_") + or normalized_key.startswith("BASH_FUNC_") + or ( + normalized_key.startswith("PYTHON") + and not ( + (normalized_key == "PYTHONNOUSERSITE" and value == "1") + or (normalized_key == "PYTHONSAFEPATH" and value == "1") + or normalized_key == "PYTHONUNBUFFERED" + ) + ) + or normalized_key + in { + "BASH_ENV", + "BASHOPTS", + "CDPATH", + "ENV", + "GLOBIGNORE", + "IFS", + "PS4", + "SHELLOPTS", + } + or normalized_key == "PROXY" + or normalized_key.endswith("_PROXY") + or normalized_key in RUNTIME_TLS_TRUST_ENVIRONMENT + for key, value in os.environ.items() + for normalized_key in (key.upper(),) + ) + if forbidden_environment: + raise SystemExit( + "Refusing inherited credential, proxy, TLS, or PostgreSQL environment in Leo runtime mode." + ) + return + + if mode != "clone-readonly": + raise SystemExit(f"Unsupported Cloud SQL credential mode: {mode}.") + canonical_db = str(args.canonical_db or "") + audit_db = str(args.db or "") + pgoptions = os.environ.get("PGOPTIONS", "") + if not re.fullmatch(r"teleo_clone_[a-z0-9_]+", canonical_db) or audit_db != canonical_db: + raise SystemExit("Clone credential mode is limited to one teleo_clone_* database.") + if args.command not in RECEIPTED_READ_COMMANDS: + raise SystemExit("Clone credential mode permits read commands only.") + if pgoptions != CLONE_READ_ONLY_PGOPTIONS: + raise SystemExit("Clone credential mode requires server-enforced default_transaction_read_only=on.") + if not user or not os.environ.get("PGPASSWORD"): + raise SystemExit("Clone credential mode requires an explicit user and inherited test credential.") + + def terms_for(query: str) -> list[str]: terms: list[str] = [] for token in re.findall(r"[A-Za-z0-9][A-Za-z0-9.+#/-]*", query.lower()): @@ -268,34 +469,145 @@ def sql_array(values: list[str], cast: str = "text") -> str: return "array[" + ",".join(sql_literal(value) for value in values) + f"]::{cast}[]" +def _read_http_response(request: urllib.request.Request, *, max_bytes: int, purpose: str) -> bytes: + try: + with RUNTIME_HTTP_OPENER.open(request, timeout=HTTP_TIMEOUT_SECONDS) as response: + if response.getcode() != 200: + raise SystemExit(f"{purpose} request failed.") + body = response.read(max_bytes + 1) + except SystemExit: + raise + # This is a credential-bearing boundary. Never propagate provider, proxy, + # TLS, or test-double exception text because it can contain bearer data. + except Exception: + raise SystemExit(f"{purpose} request failed.") from None + if not isinstance(body, bytes) or len(body) > max_bytes: + raise SystemExit(f"{purpose} response was invalid.") + return body + + +def _strict_json_object(body: bytes, *, purpose: str) -> dict[str, Any]: + def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError("duplicate JSON key") + value[key] = item + return value + + try: + decoded = body.decode("utf-8", errors="strict") + value = json.loads( + decoded, + object_pairs_hook=reject_duplicate_keys, + parse_constant=lambda _constant: (_ for _ in ()).throw(ValueError("non-finite JSON value")), + ) + except (UnicodeDecodeError, ValueError, TypeError): + raise SystemExit(f"{purpose} response was invalid.") from None + if not isinstance(value, dict): + raise SystemExit(f"{purpose} response was invalid.") + return value + + +def _metadata_request(url: str, *, max_bytes: int, purpose: str) -> bytes: + request = urllib.request.Request(url, headers=METADATA_HEADERS, method="GET") + return _read_http_response(request, max_bytes=max_bytes, purpose=purpose) + + +def runtime_password_from_metadata() -> str: + email_body = _metadata_request( + GCE_METADATA_EMAIL_URL, + max_bytes=MAX_METADATA_EMAIL_BYTES, + purpose="Runtime metadata identity", + ) + try: + service_account = email_body.decode("ascii", errors="strict") + except UnicodeDecodeError: + raise SystemExit("Runtime metadata identity response was invalid.") from None + if service_account != RUNTIME_SERVICE_ACCOUNT: + raise SystemExit("Runtime metadata identity did not match the required service account.") + + token_value = _strict_json_object( + _metadata_request( + GCE_METADATA_TOKEN_URL, + max_bytes=MAX_METADATA_TOKEN_BYTES, + purpose="Runtime metadata token", + ), + purpose="Runtime metadata token", + ) + access_token = token_value.get("access_token") + token_type = token_value.get("token_type") + expires_in = token_value.get("expires_in") + if ( + token_type != "Bearer" + or not isinstance(access_token, str) + or not re.fullmatch(r"[!-~]{1,4096}", access_token) + or not isinstance(expires_in, int) + or isinstance(expires_in, bool) + or expires_in <= 0 + ): + raise SystemExit("Runtime metadata token response was invalid.") + + secret_request = urllib.request.Request( + RUNTIME_SECRET_ACCESS_URL, + headers={"Accept": "application/json", "Authorization": f"Bearer {access_token}"}, + method="GET", + ) + secret_value = _strict_json_object( + _read_http_response( + secret_request, + max_bytes=MAX_SECRET_RESPONSE_BYTES, + purpose="Scoped runtime secret access", + ), + purpose="Scoped runtime secret access", + ) + payload = secret_value.get("payload") + encoded_secret = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(encoded_secret, str): + raise SystemExit("Scoped runtime secret response was invalid.") + try: + credential_bytes = base64.b64decode(encoded_secret, validate=True) + except (binascii.Error, ValueError): + raise SystemExit("Scoped runtime secret response was invalid.") from None + if not credential_bytes or len(credential_bytes) > MAX_RUNTIME_SECRET_BYTES: + raise SystemExit("Scoped runtime secret response was invalid.") + if any(byte in credential_bytes for byte in (0, 10, 13)): + raise SystemExit("Scoped runtime secret response was invalid.") + try: + return credential_bytes.decode("utf-8", errors="strict") + except UnicodeDecodeError: + raise SystemExit("Scoped runtime secret response was invalid.") from None + + def password(args: argparse.Namespace) -> str: + validate_runtime_connection(args) + if args.credential_mode == "runtime": + return runtime_password_from_metadata() if os.environ.get("PGPASSWORD"): return os.environ["PGPASSWORD"] - result = subprocess.run( - [ - "gcloud", - "secrets", - "versions", - "access", - "latest", - f"--secret={args.password_secret}", - f"--project={args.project}", - ], - text=True, - capture_output=True, - check=False, - ) - if result.returncode != 0: - raise SystemExit(f"Secret access failed: {result.stderr.strip()}") - return result.stdout.strip() + raise SystemExit("Clone credential mode requires an inherited test credential.") def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: - env = os.environ.copy() - env["PGPASSWORD"] = password(args) - conn = f"host={args.host} port={args.port} dbname={db or args.db} user={args.user} sslmode=require" + credential = password(args) + if args.credential_mode == "runtime": + env = dict(RUNTIME_PSQL_ENV_BASE) + env["PGSSLROOTCERT"] = runtime_ssl_root_cert() + else: + env = {key: value for key, value in os.environ.items() if not key.startswith("PG")} + env.update( + { + "PGHOST": str(args.host), + "PGPORT": str(args.port), + "PGDATABASE": str(db or args.db), + "PGUSER": str(args.user), + "PGPASSWORD": credential, + } + ) + if args.credential_mode == "clone-readonly": + env["PGOPTIONS"] = CLONE_READ_ONLY_PGOPTIONS result = subprocess.run( - ["psql", conn, "-At", "-q", "-v", "ON_ERROR_STOP=1"], + [RUNTIME_PSQL_BIN, "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"], text=True, capture_output=True, input=sql, @@ -303,9 +615,7 @@ def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: check=False, ) if result.returncode != 0: - raise SystemExit( - f"Cloud SQL query failed ({result.returncode})\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" - ) + raise SystemExit(f"Cloud SQL query failed with exit status {result.returncode}.") return result.stdout @@ -328,6 +638,10 @@ def database_read_marker(args: argparse.Namespace) -> dict[str, Any]: select jsonb_build_object( 'database', current_database(), 'database_user', current_user, + 'server_addr', inet_server_addr()::text, + 'server_port', inet_server_port(), + 'ssl', coalesce((select ssl from pg_catalog.pg_stat_ssl where pid = pg_backend_pid()), false), + 'ssl_version', (select version from pg_catalog.pg_stat_ssl where pid = pg_backend_pid()), 'system_identifier', (select system_identifier::text from pg_control_system()), 'wal_lsn', pg_current_wal_lsn()::text )::text; @@ -335,12 +649,36 @@ select jsonb_build_object( rows = psql_json_lines(args, sql, db=args.canonical_db) if len(rows) != 1: raise SystemExit("Canonical database read marker returned an invalid row count.") - return rows[0] + marker = rows[0] + expected_database = str(args.canonical_db or "") + expected_user = str(args.user or "") + if marker.get("database") != expected_database or marker.get("database_user") != expected_user: + raise SystemExit("Canonical database read marker returned an unexpected database identity.") + if getattr(args, "credential_mode", "runtime") == "runtime": + if marker.get("server_addr") != RUNTIME_CLOUDSQL_HOST: + raise SystemExit("Canonical database read marker returned an unexpected server address.") + if str(marker.get("server_port")) != RUNTIME_CLOUDSQL_PORT: + raise SystemExit("Canonical database read marker returned an unexpected server port.") + if marker.get("ssl") is not True: + raise SystemExit("Canonical database read marker did not prove an encrypted connection.") + if marker.get("system_identifier") != RUNTIME_SYSTEM_IDENTIFIER: + raise SystemExit("Canonical database read marker returned an unexpected system identifier.") + return marker def _read_marker_stable(before: dict[str, Any], after: dict[str, Any]) -> bool: return all( - before.get(key) == after.get(key) for key in ("database", "database_user", "system_identifier", "wal_lsn") + before.get(key) == after.get(key) + for key in ( + "database", + "database_user", + "server_addr", + "server_port", + "ssl", + "ssl_version", + "system_identifier", + "wal_lsn", + ) ) @@ -413,6 +751,10 @@ def build_retrieval_receipt( "attempts": attempts, "database": before.get("database"), "database_user": before.get("database_user"), + "server_addr": before.get("server_addr"), + "server_port": before.get("server_port"), + "ssl": before.get("ssl"), + "ssl_version": before.get("ssl_version"), "system_identifier": before.get("system_identifier"), "wal_lsn_before": before.get("wal_lsn"), "wal_lsn_after": after.get("wal_lsn"), @@ -868,6 +1210,11 @@ def query_rows(args: argparse.Namespace, query: str, limit: int) -> dict[str, An canonical = query_canonical_rows(args, query, limit) if canonical.get("hit_count_total", 0) or canonical.get("hits"): return canonical + if os.environ.get("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "") != "1": + canonical["canonical_fallback_reason"] = ( + "no matching canonical public/persona/strategy/belief rows; audit fallback disabled" + ) + return canonical if not audit_restore_available(args): canonical["canonical_fallback_reason"] = ( "no matching canonical public/persona/strategy/belief rows; teleo_restore audit fallback unavailable" @@ -1054,57 +1401,37 @@ def propose_core_change(args: argparse.Namespace) -> dict[str, Any]: with params as ( select {sql_literal(args.proposal_type)}::text as proposal_type, - nullif({sql_literal(args.proposed_by)}, '') as proposed_by_handle, nullif({sql_literal(args.channel)}, '') as channel, nullif({sql_literal(args.source_ref)}, '') as source_ref, {sql_literal(args.rationale)} as rationale, {sql_literal(payload_json)}::jsonb as payload ), proposer as ( - select a.id, a.handle - from public.agents a, params p - where a.handle = p.proposed_by_handle - limit 1 -), inserted as ( - insert into kb_stage.kb_proposals ( - proposal_type, - status, - proposed_by_handle, - proposed_by_agent_id, - channel, - source_ref, - rationale, - payload - ) - select + select kb_stage.stage_leoclean_proposal( p.proposal_type, - 'pending_review', - p.proposed_by_handle, - proposer.id, - coalesce(p.channel, 'telegram'), + p.channel, p.source_ref, p.rationale, p.payload + ) as proposal from params p - left join proposer on true - returning * ) select jsonb_build_object( 'artifact', 'teleo_cloudsql_kb_core_change_proposal', 'backend', {sql_literal(canonical_backend(args))}, - 'id', id::text, - 'proposal_type', proposal_type, - 'status', status, - 'proposed_by_handle', proposed_by_handle, - 'proposed_by_agent_id', proposed_by_agent_id::text, - 'channel', channel, - 'source_ref', source_ref, - 'rationale', rationale, - 'payload', payload, - 'created_at', created_at::text, + 'id', proposal->>'id', + 'proposal_type', proposal->>'proposal_type', + 'status', proposal->>'status', + 'proposed_by_handle', proposal->>'proposed_by_handle', + 'proposed_by_agent_id', proposal->>'proposed_by_agent_id', + 'channel', proposal->>'channel', + 'source_ref', proposal->>'source_ref', + 'rationale', proposal->>'rationale', + 'payload', proposal->'payload', + 'created_at', proposal->>'created_at', 'canonical_apply_done', false, 'runtime_memory_write_done', false )::text -from inserted; +from proposer; """ rows = psql_json_lines(args, sql, db=args.canonical_db) if not rows: @@ -1119,7 +1446,6 @@ with params as ( {sql_literal(args.from_claim)}::uuid as from_claim, {sql_literal(args.to_claim)}::uuid as to_claim, {sql_literal(args.edge_type)}::edge_type as edge_type, - nullif({sql_literal(args.proposed_by)}, '') as proposed_by_handle, nullif({sql_literal(args.channel)}, '') as channel, nullif({sql_literal(args.source_ref)}, '') as source_ref, {sql_literal(args.rationale)} as rationale @@ -1131,11 +1457,6 @@ with params as ( select c.id, c.text from public.claims c, params p where c.id = p.to_claim -), proposer as ( - select a.id, a.handle - from public.agents a, params p - where a.handle = p.proposed_by_handle - limit 1 ), existing_edge as ( select e.id from public.claim_edges e, params p @@ -1143,23 +1464,10 @@ with params as ( and e.to_claim = p.to_claim and e.edge_type = p.edge_type limit 1 -), inserted as ( - insert into kb_stage.kb_proposals ( - proposal_type, - status, - proposed_by_handle, - proposed_by_agent_id, - channel, - source_ref, - rationale, - payload - ) - select +), staged as ( + select kb_stage.stage_leoclean_proposal( 'add_edge', - 'pending_review', - p.proposed_by_handle, - proposer.id, - coalesce(p.channel, 'telegram'), + p.channel, p.source_ref, p.rationale, jsonb_build_object( @@ -1175,29 +1483,28 @@ with params as ( 'edge_type', p.edge_type::text ) ) + ) as proposal from params p join from_row on true join to_row on true - left join proposer on true - returning * ) select jsonb_build_object( 'artifact', 'teleo_cloudsql_kb_edge_proposal', 'backend', {sql_literal(canonical_backend(args))}, - 'id', id::text, - 'proposal_type', proposal_type, - 'status', status, - 'proposed_by_handle', proposed_by_handle, - 'proposed_by_agent_id', proposed_by_agent_id::text, - 'channel', channel, - 'source_ref', source_ref, - 'rationale', rationale, - 'payload', payload, - 'created_at', created_at::text, + 'id', proposal->>'id', + 'proposal_type', proposal->>'proposal_type', + 'status', proposal->>'status', + 'proposed_by_handle', proposal->>'proposed_by_handle', + 'proposed_by_agent_id', proposal->>'proposed_by_agent_id', + 'channel', proposal->>'channel', + 'source_ref', proposal->>'source_ref', + 'rationale', proposal->>'rationale', + 'payload', proposal->'payload', + 'created_at', proposal->>'created_at', 'canonical_apply_done', false, 'runtime_memory_write_done', false )::text -from inserted; +from staged; """ rows = psql_json_lines(args, sql, db=args.canonical_db) if not rows: @@ -1365,6 +1672,8 @@ select jsonb_build_object( def audit_restore_available(args: argparse.Namespace) -> bool: + if os.environ.get("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "") != "1": + return False sql = """ select bool_and(to_regclass(table_name) is not null) from (values @@ -1675,6 +1984,7 @@ def emit_markdown(value: dict[str, Any]) -> None: def main() -> None: args = parse_args() + validate_runtime_connection(args) read_loaders = { "search": lambda: query_rows(args, args.query, args.limit), "context": lambda: query_rows(args, args.query, args.limit), diff --git a/hermes-agent/leoclean-bin/teleo-kb b/hermes-agent/leoclean-bin/teleo-kb index 808c77f..d5b3e15 100755 --- a/hermes-agent/leoclean-bin/teleo-kb +++ b/hermes-agent/leoclean-bin/teleo-kb @@ -1,9 +1,15 @@ -#!/usr/bin/env bash +#!/bin/bash set -euo pipefail PROFILE_DIR=${HERMES_HOME:-/home/teleo/.hermes/profiles/leoclean} KB_TOOL="$PROFILE_DIR/bin/kb_tool.py" -CLOUDSQL_TOOL="$PROFILE_DIR/bin/cloudsql_memory_tool.py" +SCRIPT_PATH=${BASH_SOURCE[0]} +SCRIPT_DIR=${SCRIPT_PATH%/*} +if [ "$SCRIPT_DIR" = "$SCRIPT_PATH" ]; then + SCRIPT_DIR=. +fi +SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)" +CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py" MODE=${TELEO_KB_MODE:-auto} COMMAND=${1:-} @@ -23,14 +29,30 @@ cloudsql_args() { } run_cloudsql() { - if cloudsql_supported; then - extra_args=() - while IFS= read -r arg; do - [ -n "$arg" ] && extra_args+=("$arg") - done < <(cloudsql_args) - exec python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}" + if ! cloudsql_supported; then + echo "teleo-kb: command '$COMMAND' is not supported by the Cloud SQL backend" >&2 + exit 64 fi - exec python3 "$KB_TOOL" "$@" + if [ ! -x "$CLOUDSQL_TOOL" ]; then + echo "teleo-kb: Cloud SQL tool is missing or not executable: $CLOUDSQL_TOOL" >&2 + exit 69 + fi + if [ ! -x /usr/bin/python3 ]; then + echo "teleo-kb: required Cloud SQL dependency is unavailable: /usr/bin/python3" >&2 + exit 69 + fi + if ! command -v psql >/dev/null 2>&1; then + echo "teleo-kb: required Cloud SQL dependency is unavailable: psql" >&2 + exit 69 + fi + extra_args=() + while IFS= read -r arg; do + [ -n "$arg" ] && extra_args+=("$arg") + done < <(cloudsql_args) + if [ "${#extra_args[@]}" -gt 0 ]; then + exec /usr/bin/python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}" + fi + exec /usr/bin/python3 "$CLOUDSQL_TOOL" "$@" } case "$MODE" in @@ -38,22 +60,22 @@ case "$MODE" in run_cloudsql "$@" ;; local) - exec python3 "$KB_TOOL" --local "$@" + exec /usr/bin/python3 "$KB_TOOL" --local "$@" ;; remote) - exec python3 "$KB_TOOL" "$@" + exec /usr/bin/python3 "$KB_TOOL" "$@" ;; auto) # Canonical commands must fail closed on Cloud SQL errors. Falling through # to a different local database would make source-of-truth behavior depend - # on health-check latency. - if [ -x "$CLOUDSQL_TOOL" ] && command -v psql >/dev/null 2>&1 && command -v gcloud >/dev/null 2>&1 && cloudsql_supported; then + # on health-check latency or host package state. + if cloudsql_supported; then run_cloudsql "$@" fi if command -v docker >/dev/null 2>&1; then - exec python3 "$KB_TOOL" --local "$@" + exec /usr/bin/python3 "$KB_TOOL" --local "$@" fi - exec python3 "$KB_TOOL" "$@" + exec /usr/bin/python3 "$KB_TOOL" "$@" ;; *) echo "Unsupported TELEO_KB_MODE=$MODE; expected auto, cloudsql, local, or remote" >&2 diff --git a/hermes-agent/leoclean-bin/teleo-kb-gcp b/hermes-agent/leoclean-bin/teleo-kb-gcp new file mode 100755 index 0000000..9e56c18 --- /dev/null +++ b/hermes-agent/leoclean-bin/teleo-kb-gcp @@ -0,0 +1,52 @@ +#!/bin/bash +set -euo pipefail + +# This wrapper is installed only into the GCP service's root-controlled bind. +# Ignore per-invocation mode and endpoint overrides so a service child cannot +# select the VPS/local backend or a different database credential context. +SCRIPT_PATH=${BASH_SOURCE[0]} +SCRIPT_DIR=${SCRIPT_PATH%/*} +if [ "$SCRIPT_DIR" = "$SCRIPT_PATH" ]; then + SCRIPT_DIR=. +fi +SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)" +CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py" + +for argument in "$@"; do + case "$argument" in + --host|--host=*|--port|--port=*|--db|--db=*|--canonical-db|--canonical-db=*|\ + --user|--user=*|--password-secret|--password-secret=*|--project|--project=*|\ + --credential-mode|--credential-mode=*) + echo "teleo-kb: GCP runtime identity and endpoint overrides are forbidden" >&2 + exit 64 + ;; + esac +done + +if [ ! -x "$CLOUDSQL_TOOL" ]; then + echo "teleo-kb: reviewed GCP Cloud SQL tool is unavailable" >&2 + exit 69 +fi +if [ ! -x /usr/bin/python3 ] || [ ! -x /usr/bin/psql ]; then + echo "teleo-kb: required GCP runtime dependency is unavailable" >&2 + exit 69 +fi + +exec /usr/bin/env -i \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + LANG=C LC_ALL=C \ + PYTHONNOUSERSITE=1 PYTHONSAFEPATH=1 \ + CLOUDSDK_CONFIG=/usr/local/libexec/livingip/leoclean-kb/gcloud-config \ + PGSSLMODE=verify-ca \ + PGSSLROOTCERT=/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem \ + TELEO_KB_MODE=cloudsql \ + TELEO_GCP_METADATA_ONLY=1 \ + TELEO_GCP_PROJECT=teleo-501523 \ + TELEO_CLOUDSQL_HOST=10.61.0.3 \ + TELEO_CLOUDSQL_PORT=5432 \ + TELEO_CLOUDSQL_DB=teleo_canonical \ + TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical \ + TELEO_CLOUDSQL_USER=leoclean_kb_runtime \ + TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \ + TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \ + /usr/bin/python3 -I "$CLOUDSQL_TOOL" "$@" diff --git a/ops/gcp-teleo-pgvector-standby-server-ca.pem b/ops/gcp-teleo-pgvector-standby-server-ca.pem new file mode 100644 index 0000000..e31490d --- /dev/null +++ b/ops/gcp-teleo-pgvector-standby-server-ca.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDcTCCAlmgAwIBAgIBADANBgkqhkiG9w0BAQsFADBwMS0wKwYDVQQuEyQzNTcw +NmEzOC03MGRjLTQ3YTQtOWExOS04NGMxYTk0YzQ3ZjMxHDAaBgNVBAMTE0Nsb3Vk +IFNRTCBTZXJ2ZXIgQ0ExFDASBgNVBAoTC0dvb2dsZSwgSW5jMQswCQYDVQQGEwJV +UzAeFw0yNjA3MDcwOTM3NTJaFw0zNjA3MDQwOTM4NTJaMHAxLTArBgNVBC4TJDM1 +NzA2YTM4LTcwZGMtNDdhNC05YTE5LTg0YzFhOTRjNDdmMzEcMBoGA1UEAxMTQ2xv +dWQgU1FMIFNlcnZlciBDQTEUMBIGA1UEChMLR29vZ2xlLCBJbmMxCzAJBgNVBAYT +AlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0HwhrUzFIi9bqPbC +9FQjCalxEVYsu7tw+YzBtUaKOy+u/Y1e6TKJzFng/9wS8pXgY1Ul1/mn1S205yKR +5ydEESq7UL4pp7onSb2vGeu4NDTXDbcimZz++35dPbhFw7wtm1QSRMBHsONEDGaS +7/qIB3EN79SS9GxbSUZNsZOAn4DM9RSahtQrXa6NLlP7I9gPxvqEKRjFqUcBkWsE +Qzoz/yyxYQybZQo1UR9XCbGWFphi85cW15ITpfs7czoOVUk39bPNJKAVQFRqPUWK +xBC+1pHb7fe7xYk5j3jJbtoL6ixunQIFDCGRCy7/SEfIj5dWriDajVHwctilUmu1 +Xy/A0QIDAQABoxYwFDASBgNVHRMBAf8ECDAGAQH/AgEAMA0GCSqGSIb3DQEBCwUA +A4IBAQBi8MoiT4kJAut4mLPibKldRB9s2Of7Jw8PmKNS50BIOZf8zrSrNMX5yC2i +/y9nmc8k9OCIYbzC6F/mTACmtHM1phoBtPYDjWYp+YS9cdo0HHGU2hwEP7PTfEOg +11ua2J1gjFbgVZVb7qNGV4n1QjCcAh1hWnJ2gLa55hiZBGZ0fuHTwBagWwRtom+u +YdD1d1NETInnCxnhzu7q+r0sjFUJABIj4qU/q/uVMHaAuy4Zy4PMctQofKSJ4ixf +vbU2/Sm7cbzgzcg2TJwpSqdDM/EdDMi5JtmMyZnRoKLZjH3CYI1091dyTBdk8RhK +MB4pqjJurXS8xKeA9dQjuvnIQvot +-----END CERTIFICATE----- diff --git a/ops/gcp_leoclean_runtime_role.sql b/ops/gcp_leoclean_runtime_role.sql new file mode 100644 index 0000000..5962180 --- /dev/null +++ b/ops/gcp_leoclean_runtime_role.sql @@ -0,0 +1,1346 @@ +\set ON_ERROR_STOP on + +-- One-time GCP staging provisioning for Leo's Cloud SQL identity. Run this +-- only through ops/provision_gcp_leoclean_runtime_role.sh; psql's \password +-- command encrypts the supplied value client-side, so cleartext is never SQL. +-- +-- The login can read the canonical allowlist and can stage a pending proposal +-- through one narrowly-scoped SECURITY DEFINER function. It cannot inherit or +-- SET ROLE into another principal, forge the proposer identity, write directly +-- to canonical/staging tables, or inspect the legacy teleo_restore audit schema. +-- Keep the runtime unable to authenticate until both database phases have +-- completed and verified. Any failure before the final transaction therefore +-- leaves a disabled role instead of a partially provisioned login. +select 'create role leoclean_kb_runtime nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8' +where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_runtime') +\gexec + +select 'create role leoclean_kb_stage_owner nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1' +where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_stage_owner') +\gexec + +alter role leoclean_kb_runtime + with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; +alter role leoclean_kb_stage_owner + with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1 + password null; + +-- Snapshot every non-superuser principal connected to either scoped role before +-- removing the direct membership edges. Existing sessions can retain an +-- already-selected SET ROLE identity after NOLOGIN/revocation, so all other +-- connected sessions must be terminated before password or privilege work. +create temporary table leoclean_fenced_roles ( + role_oid oid primary key, + role_name name unique not null +) on commit preserve rows; + +with recursive connected(role_oid) as ( + values + ('leoclean_kb_runtime'::pg_catalog.regrole::oid), + ('leoclean_kb_stage_owner'::pg_catalog.regrole::oid) + union + select membership.member + from connected + join pg_catalog.pg_auth_members membership + on membership.roleid = connected.role_oid +) +insert into pg_temp.leoclean_fenced_roles (role_oid, role_name) +select role_row.oid, role_row.rolname + from connected + join pg_catalog.pg_roles role_row on role_row.oid = connected.role_oid + where not role_row.rolsuper; + +-- NOINHERIT does not prevent SET ROLE. Remove every direct membership edge in +-- either direction before terminating the snapshotted sessions, closing the +-- race for newly connected membership principals. +select format('revoke %I from %I', granted_role.rolname, member_role.rolname) + from pg_catalog.pg_auth_members membership + join pg_catalog.pg_roles granted_role on granted_role.oid = membership.roleid + join pg_catalog.pg_roles member_role on member_role.oid = membership.member + where granted_role.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + or member_role.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') +\gexec + +do $session_fence$ +declare + target record; + remaining text; +begin + for target in + select activity.pid + from pg_catalog.pg_stat_activity activity + join pg_temp.leoclean_fenced_roles fenced on fenced.role_name = activity.usename + where activity.pid <> pg_catalog.pg_backend_pid() + and activity.backend_type = 'client backend' + loop + if not pg_catalog.pg_terminate_backend(target.pid, 5000) then + raise exception 'could not terminate a pre-existing scoped-role session'; + end if; + end loop; + + select pg_catalog.string_agg(activity.pid::text, ',' order by activity.pid) + into remaining + from pg_catalog.pg_stat_activity activity + join pg_temp.leoclean_fenced_roles fenced on fenced.role_name = activity.usename + where activity.pid <> pg_catalog.pg_backend_pid() + and activity.backend_type = 'client backend'; + if remaining is not null then + raise exception 'pre-existing scoped-role sessions remain after termination'; + end if; + + select pg_catalog.string_agg(prepared.gid, ',' order by prepared.gid) + into remaining + from pg_catalog.pg_prepared_xacts prepared + join pg_temp.leoclean_fenced_roles fenced on fenced.role_name = prepared.owner; + if remaining is not null then + raise exception 'scoped-role prepared transactions must be resolved before provisioning'; + end if; +end +$session_fence$; + +-- This is deliberately after the autocommitted NOLOGIN/session/membership +-- fence. Interruption can leave a disabled role, never a still-enabled old +-- login or an old connected session. The wrapper supplies the two prompts. +\password leoclean_kb_runtime + +alter role leoclean_kb_runtime reset all; +alter role leoclean_kb_stage_owner reset all; +drop table pg_temp.leoclean_fenced_roles; + +select pg_catalog.format( + 'alter role %I in database %I reset all', + role_row.rolname, + database_row.datname + ) + from pg_catalog.pg_db_role_setting setting_row + join pg_catalog.pg_roles role_row on role_row.oid = setting_row.setrole + join pg_catalog.pg_database database_row on database_row.oid = setting_row.setdatabase + where role_row.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') +\gexec + +-- PostgreSQL normally grants TEMP to PUBLIC. A role-specific REVOKE cannot +-- override that inherited grant, while REVOKE TEMP FROM PUBLIC would be a +-- database-wide change that could break kb_apply/review or operator workflows. +-- Leave PUBLIC TEMP policy unchanged here and surface it in the final receipt. +-- TEMP cannot shadow this SECURITY DEFINER boundary: pg_catalog is first, +-- pg_temp is explicitly last, and every relation reference is schema-qualified. +-- Database CREATE, direct TEMP grants, and all unexpected direct database ACLs +-- are rejected below. A database-wide TEMP removal requires a separate role-use +-- inventory and explicit grants to every workload that still needs it. + +alter role leoclean_kb_runtime in database teleo_canonical reset all; +alter role leoclean_kb_stage_owner in database teleo_canonical reset all; +alter role leoclean_kb_runtime in database teleo_kb reset all; +alter role leoclean_kb_stage_owner in database teleo_kb reset all; + +alter role leoclean_kb_runtime in database teleo_canonical + set search_path = pg_catalog, public, kb_stage; +alter role leoclean_kb_runtime in database teleo_canonical + set statement_timeout = '15s'; +alter role leoclean_kb_runtime in database teleo_canonical + set lock_timeout = '2s'; + +\connect teleo_canonical + +begin; + +-- CONNECT granted to PUBLIC cannot be denied to one role with a role-specific +-- REVOKE. Preserve every existing non-scoped principal's effective CONNECT +-- set as explicit ACLs, remove the cluster-wide PUBLIC grant from every +-- database (including templates), and then grant the runtime its sole target. +-- This runs in the same transaction as every remaining privilege/topology +-- assertion, so a later failure restores the original ACLs atomically. +create temporary table leoclean_preserved_database_connect ( + role_name name not null, + database_name name not null, + primary key (role_name, database_name) +) on commit drop; + +insert into pg_temp.leoclean_preserved_database_connect (role_name, database_name) +select role_row.rolname, database_row.datname + from pg_catalog.pg_roles role_row + cross join pg_catalog.pg_database database_row + where role_row.rolname not in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and pg_catalog.has_database_privilege(role_row.oid, database_row.oid, 'CONNECT'); + +select pg_catalog.format('revoke connect on database %I from public', database_row.datname) + from pg_catalog.pg_database database_row +\gexec + +select pg_catalog.format( + 'revoke all privileges on database %I from leoclean_kb_runtime, leoclean_kb_stage_owner', + database_row.datname + ) + from pg_catalog.pg_database database_row +\gexec + +select pg_catalog.format( + 'grant connect on database %I to %I', + preserved.database_name, + preserved.role_name + ) + from pg_temp.leoclean_preserved_database_connect preserved + order by preserved.database_name, preserved.role_name +\gexec + +grant connect on database teleo_canonical to leoclean_kb_runtime; + +-- The deploy principal needs temporary ownership authority to replace an +-- existing function on idempotent reruns. This membership is transaction-local +-- and is revoked before verification/commit. +select format('grant leoclean_kb_stage_owner to %I', current_user) + where current_user <> 'leoclean_kb_stage_owner' +\gexec + +revoke all on schema public, kb_stage + from leoclean_kb_runtime, leoclean_kb_stage_owner; +revoke all privileges on all tables in schema public, kb_stage + from leoclean_kb_runtime, leoclean_kb_stage_owner; +revoke all privileges on all sequences in schema public, kb_stage + from leoclean_kb_runtime, leoclean_kb_stage_owner; +revoke all privileges on all functions in schema public, kb_stage + from leoclean_kb_runtime, leoclean_kb_stage_owner; +revoke all privileges on all procedures in schema public, kb_stage + from leoclean_kb_runtime, leoclean_kb_stage_owner; + +-- Built-in defaults also grant PUBLIC EXECUTE on newly created application +-- routines. Preserve every existing non-scoped role's effective access, +-- remove PUBLIC/scoped execution from the complete application routine set, +-- and restore the inventory explicitly. The reviewed staging function is +-- normalized again after CREATE OR REPLACE below. +create temporary table leoclean_preserved_app_routine_execute ( + role_name name not null, + function_oid oid not null, + primary key (role_name, function_oid) +) on commit drop; + +insert into pg_temp.leoclean_preserved_app_routine_execute (role_name, function_oid) +select role_row.rolname, function_row.oid + from pg_catalog.pg_roles role_row + cross join pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where role_row.rolname not in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and namespace.nspname in ('public', 'kb_stage') + and pg_catalog.has_function_privilege(role_row.oid, function_row.oid, 'EXECUTE'); + +select pg_catalog.format( + 'revoke execute on %s %I.%I(%s) from public, leoclean_kb_runtime, leoclean_kb_stage_owner', + case when function_row.prokind = 'p' then 'procedure' else 'function' end, + namespace.nspname, + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid) + ) + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname in ('public', 'kb_stage') +\gexec + +select pg_catalog.format( + 'grant execute on %s %I.%I(%s) to %I', + case when function_row.prokind = 'p' then 'procedure' else 'function' end, + namespace.nspname, + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid), + preserved.role_name + ) + from pg_temp.leoclean_preserved_app_routine_execute preserved + join pg_catalog.pg_proc function_row on function_row.oid = preserved.function_oid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + order by namespace.nspname, function_row.proname, preserved.role_name +\gexec + +select pg_catalog.format( + 'revoke all privileges on large object %s from leoclean_kb_runtime, leoclean_kb_stage_owner', + metadata.oid + ) + from pg_catalog.pg_largeobject_metadata metadata +\gexec + +-- Default PostgreSQL grants EXECUTE on built-in large-object mutators to +-- PUBLIC, which would let the runtime persist data despite all relation ACLs. +-- Snapshot and explicitly preserve every existing non-scoped role's effective +-- access, then remove PUBLIC and both scoped roles from every mutator overload. +create temporary table leoclean_preserved_lo_execute ( + role_name name not null, + function_oid oid not null, + primary key (role_name, function_oid) +) on commit drop; + +insert into pg_temp.leoclean_preserved_lo_execute (role_name, function_oid) +select role_row.rolname, function_row.oid + from pg_catalog.pg_roles role_row + cross join pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where role_row.rolname not in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and pg_catalog.has_function_privilege(role_row.oid, function_row.oid, 'EXECUTE'); + +select pg_catalog.format( + 'revoke execute on function pg_catalog.%I(%s) from public, leoclean_kb_runtime, leoclean_kb_stage_owner', + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid) + ) + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) +\gexec + +select pg_catalog.format( + 'grant execute on function pg_catalog.%I(%s) to %I', + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid), + preserved.role_name + ) + from pg_temp.leoclean_preserved_lo_execute preserved + join pg_catalog.pg_proc function_row on function_row.oid = preserved.function_oid + order by function_row.proname, preserved.role_name +\gexec + +select pg_catalog.format( + 'revoke all privileges on parameter %I from leoclean_kb_runtime, leoclean_kb_stage_owner', + parameter_acl.parname + ) + from pg_catalog.pg_parameter_acl parameter_acl +\gexec + +-- Table-level REVOKE does not clear column ACLs. Clear every column ACL for both +-- principals before rebuilding the exact allowlist below. +select format( + 'revoke all privileges (%s) on table %I.%I from %I', + pg_catalog.string_agg(pg_catalog.quote_ident(attribute.attname), ', ' order by attribute.attnum), + namespace.nspname, + relation.relname, + target.rolname + ) + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join ( + values ('leoclean_kb_runtime'), ('leoclean_kb_stage_owner') + ) as target(rolname) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + group by namespace.nspname, relation.relname, target.rolname +\gexec + +grant usage on schema public, kb_stage to leoclean_kb_runtime; + +grant select on + public.agents, + public.claims, + public.claim_evidence, + public.claim_edges, + public.sources, + public.personas, + public.strategies, + public.beliefs, + public.blindspots, + public.agent_roles, + public.peer_models, + public.behavioral_rules, + public.contributor_rules, + public.reasoning_tools, + public.governance_gates, + kb_stage.kb_proposals +to leoclean_kb_runtime; + +-- The function owner is a dedicated NOLOGIN role. Grant only the columns the +-- function reads/inserts. CREATE is temporary and is revoked immediately after +-- ownership is assigned. +grant usage, create on schema kb_stage to leoclean_kb_stage_owner; +grant usage on schema public to leoclean_kb_stage_owner; +grant select (id, handle) on public.agents to leoclean_kb_stage_owner; +grant select on kb_stage.kb_proposals to leoclean_kb_stage_owner; +grant insert ( + proposal_type, + status, + proposed_by_handle, + proposed_by_agent_id, + channel, + source_ref, + rationale, + payload +) on kb_stage.kb_proposals to leoclean_kb_stage_owner; + +-- Remove the prior caller-supplied-identity overload before installing the +-- session-bound form. A runtime caller can only stage as canonical agent Leo. +drop function if exists kb_stage.stage_leoclean_proposal(text, text, text, text, text, jsonb); + +create or replace function kb_stage.stage_leoclean_proposal( + p_proposal_type text, + p_channel text, + p_source_ref text, + p_rationale text, + p_payload jsonb +) returns jsonb +language plpgsql +security definer +set search_path = pg_catalog, pg_temp +as $function$ +declare + staged kb_stage.kb_proposals%rowtype; + proposer_id uuid; +begin + if session_user <> 'leoclean_kb_runtime' then + raise exception 'stage_leoclean_proposal is restricted to leoclean_kb_runtime' + using errcode = '42501'; + end if; + if p_proposal_type not in ('revise_claim', 'revise_strategy', 'add_edge') then + raise exception 'unsupported Leo proposal type: %', p_proposal_type using errcode = '22023'; + end if; + if nullif(btrim(p_source_ref), '') is null then + raise exception 'source_ref is required' using errcode = '22023'; + end if; + if nullif(btrim(p_rationale), '') is null then + raise exception 'rationale is required' using errcode = '22023'; + end if; + if p_payload is null or jsonb_typeof(p_payload) <> 'object' then + raise exception 'proposal payload must be a JSON object' using errcode = '22023'; + end if; + + select agent.id + into proposer_id + from public.agents as agent + where agent.handle = 'leo'; + + if proposer_id is null then + raise exception 'canonical Leo agent row is required' using errcode = '23503'; + end if; + + insert into kb_stage.kb_proposals ( + proposal_type, + status, + proposed_by_handle, + proposed_by_agent_id, + channel, + source_ref, + rationale, + payload + ) values ( + p_proposal_type, + 'pending_review', + 'leo', + proposer_id, + coalesce(nullif(p_channel, ''), 'telegram'), + p_source_ref, + p_rationale, + p_payload + ) + returning * into staged; + + return to_jsonb(staged); +end +$function$; + +alter function kb_stage.stage_leoclean_proposal(text, text, text, text, jsonb) + owner to leoclean_kb_stage_owner; +revoke create on schema kb_stage from leoclean_kb_stage_owner; + +-- Normalize explicit ACL entries left by CREATE OR REPLACE or an earlier grant. +revoke all on function kb_stage.stage_leoclean_proposal(text, text, text, text, jsonb) + from public; +select format( + 'revoke all on function kb_stage.stage_leoclean_proposal(text, text, text, text, jsonb) from %I', + grantee.rolname + ) + from pg_catalog.pg_proc function_row + cross join lateral pg_catalog.aclexplode( + coalesce( + function_row.proacl, + pg_catalog.acldefault('f', function_row.proowner) + ) + ) as acl + join pg_catalog.pg_roles grantee on grantee.oid = acl.grantee + where function_row.oid = 'kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)'::pg_catalog.regprocedure + and grantee.rolname <> 'leoclean_kb_stage_owner' +\gexec + +grant execute on function kb_stage.stage_leoclean_proposal(text, text, text, text, jsonb) + to leoclean_kb_runtime; + +select format('revoke leoclean_kb_stage_owner from %I', current_user) + where current_user <> 'leoclean_kb_stage_owner' +\gexec + +do $verification$ +declare + unexpected text; + runtime_oid oid := 'leoclean_kb_runtime'::pg_catalog.regrole::oid; + owner_oid oid := 'leoclean_kb_stage_owner'::pg_catalog.regrole::oid; + stage_function oid := 'kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)'::pg_catalog.regprocedure::oid; +begin + if exists ( + select 1 + from pg_catalog.pg_roles role_row + where role_row.rolname = 'leoclean_kb_runtime' + and (role_row.rolcanlogin + or role_row.rolsuper + or role_row.rolcreatedb + or role_row.rolcreaterole + or role_row.rolinherit + or role_row.rolreplication + or role_row.rolbypassrls + or role_row.rolconnlimit <> 8) + ) then + raise exception 'leoclean_kb_runtime has unsafe provisioning role attributes'; + end if; + + if exists ( + select 1 + from pg_catalog.pg_roles role_row + where role_row.rolname = 'leoclean_kb_stage_owner' + and (role_row.rolcanlogin + or role_row.rolsuper + or role_row.rolcreatedb + or role_row.rolcreaterole + or role_row.rolinherit + or role_row.rolreplication + or role_row.rolbypassrls + or role_row.rolconnlimit <> -1) + ) then + raise exception 'leoclean_kb_stage_owner has unsafe role attributes'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I->%I', granted_role.rolname, member_role.rolname), + ', ' + order by granted_role.rolname, member_role.rolname + ) + into unexpected + from pg_catalog.pg_auth_members membership + join pg_catalog.pg_roles granted_role on granted_role.oid = membership.roleid + join pg_catalog.pg_roles member_role on member_role.oid = membership.member + where membership.roleid in (runtime_oid, owner_oid) + or membership.member in (runtime_oid, owner_oid); + if unexpected is not null then + raise exception 'scoped Leo roles retain role memberships: %', unexpected; + end if; + + if not exists ( + select 1 + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' + and relation.relkind = 'r' + and relation.relpersistence = 'p' + and not relation.relrowsecurity + and not relation.relforcerowsecurity + ) then + raise exception 'kb_stage.kb_proposals is not the reviewed permanent base table'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, relation.relkind), + ', ' + order by namespace.nspname, relation.relname + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where (namespace.nspname, relation.relname) in ( + ('public', 'agents'), ('public', 'claims'), ('public', 'claim_evidence'), + ('public', 'claim_edges'), ('public', 'sources'), ('public', 'personas'), + ('public', 'strategies'), ('public', 'beliefs'), ('public', 'blindspots'), + ('public', 'agent_roles'), ('public', 'peer_models'), ('public', 'behavioral_rules'), + ('public', 'contributor_rules'), ('public', 'reasoning_tools'), + ('public', 'governance_gates'), ('kb_stage', 'kb_proposals') + ) + and relation.relkind <> 'r'; + if unexpected is not null then + raise exception 'Leo read allowlist contains a non-base relation: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s->%s', inheritance.inhparent::pg_catalog.regclass, inheritance.inhrelid::pg_catalog.regclass), + ', ' + ) + into unexpected + from pg_catalog.pg_inherits inheritance + where inheritance.inhparent in ( + 'public.agents'::pg_catalog.regclass, 'public.claims'::pg_catalog.regclass, + 'public.claim_evidence'::pg_catalog.regclass, 'public.claim_edges'::pg_catalog.regclass, + 'public.sources'::pg_catalog.regclass, 'public.personas'::pg_catalog.regclass, + 'public.strategies'::pg_catalog.regclass, 'public.beliefs'::pg_catalog.regclass, + 'public.blindspots'::pg_catalog.regclass, 'public.agent_roles'::pg_catalog.regclass, + 'public.peer_models'::pg_catalog.regclass, 'public.behavioral_rules'::pg_catalog.regclass, + 'public.contributor_rules'::pg_catalog.regclass, 'public.reasoning_tools'::pg_catalog.regclass, + 'public.governance_gates'::pg_catalog.regclass, 'kb_stage.kb_proposals'::pg_catalog.regclass + ) + or inheritance.inhrelid in ( + 'public.agents'::pg_catalog.regclass, 'public.claims'::pg_catalog.regclass, + 'public.claim_evidence'::pg_catalog.regclass, 'public.claim_edges'::pg_catalog.regclass, + 'public.sources'::pg_catalog.regclass, 'public.personas'::pg_catalog.regclass, + 'public.strategies'::pg_catalog.regclass, 'public.beliefs'::pg_catalog.regclass, + 'public.blindspots'::pg_catalog.regclass, 'public.agent_roles'::pg_catalog.regclass, + 'public.peer_models'::pg_catalog.regclass, 'public.behavioral_rules'::pg_catalog.regclass, + 'public.contributor_rules'::pg_catalog.regclass, 'public.reasoning_tools'::pg_catalog.regclass, + 'public.governance_gates'::pg_catalog.regclass, 'kb_stage.kb_proposals'::pg_catalog.regclass + ); + if unexpected is not null then + raise exception 'Leo read allowlist has inheritance edges: %', unexpected; + end if; + + with expected(ordinal, attname, typname, not_null, has_default, collated) as ( + values + (1, 'id', 'uuid', true, true, false), + (2, 'proposal_type', 'text', true, false, true), + (3, 'status', 'text', true, true, true), + (4, 'proposed_by_handle', 'text', false, false, true), + (5, 'proposed_by_agent_id', 'uuid', false, false, false), + (6, 'channel', 'text', true, true, true), + (7, 'source_ref', 'text', false, false, true), + (8, 'rationale', 'text', true, false, true), + (9, 'payload', 'jsonb', true, false, false), + (10, 'reviewed_by_handle', 'text', false, false, true), + (11, 'reviewed_by_agent_id', 'uuid', false, false, false), + (12, 'reviewed_at', 'timestamptz', false, false, false), + (13, 'review_note', 'text', false, false, true), + (14, 'applied_by_handle', 'text', false, false, true), + (15, 'applied_by_agent_id', 'uuid', false, false, false), + (16, 'applied_at', 'timestamptz', false, false, false), + (17, 'created_at', 'timestamptz', true, true, false), + (18, 'updated_at', 'timestamptz', true, true, false) + ), actual as ( + select attribute.attnum::int as ordinal, + attribute.attname, + type_row.typname, + attribute.attnotnull as not_null, + attribute.atthasdef as has_default, + attribute.attidentity, + attribute.attgenerated, + attribute.atttypmod, + type_namespace.nspname as type_namespace, + attribute.attcollation = 'pg_catalog.default'::pg_catalog.regcollation as collated + from pg_catalog.pg_attribute attribute + join pg_catalog.pg_type type_row on type_row.oid = attribute.atttypid + join pg_catalog.pg_namespace type_namespace on type_namespace.oid = type_row.typnamespace + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attnum > 0 + and not attribute.attisdropped + ) + select pg_catalog.string_agg(coalesce(expected.attname, actual.attname), ', ' order by coalesce(expected.ordinal, actual.ordinal)) + into unexpected + from expected + full join actual using (ordinal, attname) + where expected.ordinal is null + or actual.ordinal is null + or expected.typname is distinct from actual.typname + or expected.not_null is distinct from actual.not_null + or expected.has_default is distinct from actual.has_default + or expected.collated is distinct from actual.collated + or actual.attidentity <> '' + or actual.attgenerated <> '' + or actual.atttypmod <> -1 + or actual.type_namespace <> 'pg_catalog'; + if unexpected is not null then + raise exception 'kb_stage.kb_proposals column contract drifted: %', unexpected; + end if; + + if exists ( + select 1 + from pg_catalog.pg_trigger trigger_row + where trigger_row.tgrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and not trigger_row.tgisinternal + ) or exists ( + select 1 + from pg_catalog.pg_rewrite rewrite_row + where rewrite_row.ev_class = 'kb_stage.kb_proposals'::pg_catalog.regclass + ) or exists ( + select 1 + from pg_catalog.pg_policy policy_row + where policy_row.polrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + ) or exists ( + select 1 + from pg_catalog.pg_attribute attribute + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attnum > 0 + and not attribute.attisdropped + and attribute.attgenerated <> '' + ) then + raise exception 'kb_stage.kb_proposals has unsafe trigger/rule/RLS/generated topology'; + end if; + + with expected(attname, expression) as ( + values + ('id', 'gen_random_uuid()'), + ('status', '''pending_review''::text'), + ('channel', '''cli''::text'), + ('created_at', 'now()'), + ('updated_at', 'now()') + ), actual as ( + select attribute.attname, + pg_catalog.pg_get_expr(default_row.adbin, default_row.adrelid) as expression + from pg_catalog.pg_attribute attribute + join pg_catalog.pg_attrdef default_row + on default_row.adrelid = attribute.attrelid + and default_row.adnum = attribute.attnum + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attnum > 0 + and not attribute.attisdropped + ) + select pg_catalog.string_agg(coalesce(expected.attname, actual.attname), ', ') + into unexpected + from expected + full join actual using (attname) + where expected.expression is distinct from actual.expression; + if unexpected is not null then + raise exception 'kb_stage.kb_proposals default topology drifted: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s:%s', dependency.refclassid::pg_catalog.regclass, dependency.refobjid), + ', ' + ) + into unexpected + from pg_catalog.pg_attrdef default_row + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + where default_row.adrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and ( + dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + and dependency.refobjid not in ( + 'pg_catalog.gen_random_uuid()'::pg_catalog.regprocedure, + 'pg_catalog.now()'::pg_catalog.regprocedure + ) + or dependency.refclassid = 'pg_catalog.pg_type'::pg_catalog.regclass + and not exists ( + select 1 + from pg_catalog.pg_type type_row + join pg_catalog.pg_namespace namespace on namespace.oid = type_row.typnamespace + where type_row.oid = dependency.refobjid + and namespace.nspname = 'pg_catalog' + ) + or dependency.refclassid = 'pg_catalog.pg_operator'::pg_catalog.regclass + and not exists ( + select 1 + from pg_catalog.pg_operator operator_row + join pg_catalog.pg_namespace namespace on namespace.oid = operator_row.oprnamespace + where operator_row.oid = dependency.refobjid + and namespace.nspname = 'pg_catalog' + ) + or dependency.refclassid = 'pg_catalog.pg_collation'::pg_catalog.regclass + and not exists ( + select 1 + from pg_catalog.pg_collation collation_row + join pg_catalog.pg_namespace namespace on namespace.oid = collation_row.collnamespace + where collation_row.oid = dependency.refobjid + and namespace.nspname = 'pg_catalog' + ) + ); + if unexpected is not null then + raise exception 'kb_stage.kb_proposals defaults depend on unreviewed objects: %', unexpected; + end if; + + if ( + select pg_catalog.count(*) <> 6 + or pg_catalog.count(*) filter ( + where (constraint_row.conname, constraint_row.contype) in ( + ('kb_proposals_pkey', 'p'), + ('kb_proposals_proposal_type_check', 'c'), + ('kb_proposals_status_check', 'c'), + ('kb_proposals_applied_by_agent_id_fkey', 'f'), + ('kb_proposals_proposed_by_agent_id_fkey', 'f'), + ('kb_proposals_reviewed_by_agent_id_fkey', 'f') + ) + ) <> 6 + from pg_catalog.pg_constraint constraint_row + where constraint_row.conrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + ) or exists ( + select 1 + from pg_catalog.pg_constraint constraint_row + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_constraint'::pg_catalog.regclass + and dependency.objid = constraint_row.oid + and dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + join pg_catalog.pg_proc function_row on function_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where constraint_row.conrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and namespace.nspname <> 'pg_catalog' + ) or exists ( + select 1 + from pg_catalog.pg_index index_row + where index_row.indrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and ( + index_row.indexprs is not null + or ( + index_row.indpred is not null + and pg_catalog.pg_get_expr(index_row.indpred, index_row.indrelid) + <> '(proposed_by_handle IS NOT NULL)' + ) + ) + ) then + raise exception 'kb_stage.kb_proposals constraint/index topology drifted'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format( + '%I:%s:%s:grantable=%s', + database_row.datname, + grantee.rolname, + acl.privilege_type, + acl.is_grantable + ), + ', ' + order by database_row.datname, grantee.rolname, acl.privilege_type + ) + into unexpected + from pg_catalog.pg_database database_row + cross join lateral pg_catalog.aclexplode( + coalesce( + database_row.datacl, + pg_catalog.acldefault('d', database_row.datdba) + ) + ) as acl + join pg_catalog.pg_roles grantee on grantee.oid = acl.grantee + where grantee.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and not ( + database_row.datname = 'teleo_canonical' + and grantee.rolname = 'leoclean_kb_runtime' + and acl.privilege_type = 'CONNECT' + and not acl.is_grantable + ); + if unexpected is not null then + raise exception 'scoped Leo roles retain unexpected direct database ACLs: %', unexpected; + end if; + + select pg_catalog.string_agg(database_row.datname, ', ' order by database_row.datname) + into unexpected + from pg_catalog.pg_database database_row + where database_row.datname <> 'teleo_canonical' + and has_database_privilege('leoclean_kb_runtime', database_row.oid, 'CONNECT'); + if unexpected is not null then + raise exception 'runtime CONNECT reaches a noncanonical database: %', unexpected; + end if; + + select pg_catalog.string_agg(database_row.datname, ', ' order by database_row.datname) + into unexpected + from pg_catalog.pg_database database_row + where has_database_privilege('leoclean_kb_stage_owner', database_row.oid, 'CONNECT'); + if unexpected is not null then + raise exception 'stage owner CONNECT reaches a database: %', unexpected; + end if; + + select pg_catalog.string_agg(database_row.datname, ', ' order by database_row.datname) + into unexpected + from pg_catalog.pg_database database_row + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where acl.grantee = 0 + and acl.privilege_type = 'CONNECT'; + if unexpected is not null then + raise exception 'PUBLIC retains database CONNECT: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s:%s', setting_row.setdatabase, setting_row.setconfig), + ', ' + ) + into unexpected + from pg_catalog.pg_db_role_setting setting_row + where setting_row.setrole in (runtime_oid, owner_oid) + and not ( + setting_row.setrole = runtime_oid + and setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = 'teleo_canonical' + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ); + if unexpected is not null or not exists ( + select 1 + from pg_catalog.pg_db_role_setting setting_row + where setting_row.setrole = runtime_oid + and setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = 'teleo_canonical' + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ) then + raise exception 'scoped Leo role settings differ from the reviewed contract'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I:%s', database_row.datname, scoped_role.name), + ', ' + order by database_row.datname, scoped_role.name + ) + into unexpected + from pg_catalog.pg_database database_row + cross join (values ('leoclean_kb_runtime'), ('leoclean_kb_stage_owner')) scoped_role(name) + where has_database_privilege(scoped_role.name, database_row.oid, 'CREATE'); + if unexpected is not null then + raise exception 'a scoped Leo role unexpectedly has database CREATE: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s:%s:%s', dependency.classid::pg_catalog.regclass, dependency.objid, dependency.objsubid), + ', ' + ) + into unexpected + from pg_catalog.pg_shdepend dependency + where dependency.refclassid = 'pg_catalog.pg_authid'::pg_catalog.regclass + and dependency.refobjid = runtime_oid + and dependency.deptype = 'o'; + if unexpected is not null then + raise exception 'leoclean_kb_runtime unexpectedly owns database objects: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s:%s:%s', dependency.classid::pg_catalog.regclass, dependency.objid, dependency.objsubid), + ', ' + ) + into unexpected + from pg_catalog.pg_shdepend dependency + where dependency.refclassid = 'pg_catalog.pg_authid'::pg_catalog.regclass + and dependency.refobjid = owner_oid + and dependency.deptype = 'o' + and not ( + dependency.dbid = (select database_row.oid from pg_catalog.pg_database database_row where database_row.datname = current_database()) + and + dependency.classid = 'pg_catalog.pg_proc'::pg_catalog.regclass + and dependency.objid = stage_function + and dependency.objsubid = 0 + ); + if unexpected is not null then + raise exception 'leoclean_kb_stage_owner owns unexpected database objects: %', unexpected; + end if; + + select pg_catalog.string_agg(metadata.oid::text, ', ' order by metadata.oid) + into unexpected + from pg_catalog.pg_largeobject_metadata metadata + where metadata.lomowner in (runtime_oid, owner_oid) + or exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(metadata.lomacl, pg_catalog.acldefault('L', metadata.lomowner)) + ) acl + where acl.grantee in (0, runtime_oid, owner_oid) + and acl.privilege_type in ('SELECT', 'UPDATE') + ); + if unexpected is not null then + raise exception 'scoped Leo roles retain large-object ownership or ACL reachability: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I(%s)', function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid)), + ', ' + ) + into unexpected + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and ( + has_function_privilege('leoclean_kb_runtime', function_row.oid, 'EXECUTE') + or has_function_privilege('leoclean_kb_stage_owner', function_row.oid, 'EXECUTE') + ); + if unexpected is not null then + raise exception 'PUBLIC or inherited large-object mutators remain executable: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I(%s)', function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid)), + ', ' + order by function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid) + ) + into unexpected + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + cross join lateral pg_catalog.aclexplode( + coalesce(function_row.proacl, pg_catalog.acldefault('f', function_row.proowner)) + ) acl + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and acl.grantee = 0 + and acl.privilege_type = 'EXECUTE'; + if unexpected is not null then + raise exception 'PUBLIC retains large-object mutator EXECUTE: %', unexpected; + end if; + + select pg_catalog.string_agg(parameter_acl.parname, ', ' order by parameter_acl.parname) + into unexpected + from pg_catalog.pg_parameter_acl parameter_acl + where has_parameter_privilege('leoclean_kb_runtime', parameter_acl.parname, 'SET') + or has_parameter_privilege('leoclean_kb_runtime', parameter_acl.parname, 'ALTER SYSTEM') + or has_parameter_privilege('leoclean_kb_stage_owner', parameter_acl.parname, 'SET') + or has_parameter_privilege('leoclean_kb_stage_owner', parameter_acl.parname, 'ALTER SYSTEM'); + if unexpected is not null then + raise exception 'scoped Leo roles retain parameter privileges: %', unexpected; + end if; + + if has_schema_privilege('leoclean_kb_runtime', 'public', 'CREATE') + or has_schema_privilege('leoclean_kb_runtime', 'kb_stage', 'CREATE') + or has_schema_privilege('leoclean_kb_stage_owner', 'public', 'CREATE') + or has_schema_privilege('leoclean_kb_stage_owner', 'kb_stage', 'CREATE') then + raise exception 'a scoped Leo role unexpectedly has schema CREATE'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name), + ', ' + order by namespace.nspname, relation.relname, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and has_table_privilege('leoclean_kb_runtime', relation.oid, privilege.name); + if unexpected is not null then + raise exception 'leoclean_kb_runtime unexpectedly has table-level DML: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I:%s', namespace.nspname, relation.relname, attribute.attname, privilege.name), + ', ' + order by namespace.nspname, relation.relname, attribute.attnum, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege('leoclean_kb_runtime', relation.oid, attribute.attnum, privilege.name); + if unexpected is not null then + raise exception 'leoclean_kb_runtime unexpectedly has column-level DML: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I', namespace.nspname, relation.relname), + ', ' + order by namespace.nspname, relation.relname + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and has_table_privilege('leoclean_kb_runtime', relation.oid, 'SELECT') + and (namespace.nspname, relation.relname) not in ( + ('public', 'agents'), + ('public', 'claims'), + ('public', 'claim_evidence'), + ('public', 'claim_edges'), + ('public', 'sources'), + ('public', 'personas'), + ('public', 'strategies'), + ('public', 'beliefs'), + ('public', 'blindspots'), + ('public', 'agent_roles'), + ('public', 'peer_models'), + ('public', 'behavioral_rules'), + ('public', 'contributor_rules'), + ('public', 'reasoning_tools'), + ('public', 'governance_gates'), + ('kb_stage', 'kb_proposals') + ); + if unexpected is not null then + raise exception 'leoclean_kb_runtime can SELECT outside its table allowlist: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I', namespace.nspname, relation.relname, attribute.attname), + ', ' + order by namespace.nspname, relation.relname, attribute.attnum + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege('leoclean_kb_runtime', relation.oid, attribute.attnum, 'SELECT') + and (namespace.nspname, relation.relname) not in ( + ('public', 'agents'), + ('public', 'claims'), + ('public', 'claim_evidence'), + ('public', 'claim_edges'), + ('public', 'sources'), + ('public', 'personas'), + ('public', 'strategies'), + ('public', 'beliefs'), + ('public', 'blindspots'), + ('public', 'agent_roles'), + ('public', 'peer_models'), + ('public', 'behavioral_rules'), + ('public', 'contributor_rules'), + ('public', 'reasoning_tools'), + ('public', 'governance_gates'), + ('kb_stage', 'kb_proposals') + ); + if unexpected is not null then + raise exception 'leoclean_kb_runtime has column SELECT outside its allowlist: %', unexpected; + end if; + + if has_table_privilege('leoclean_kb_stage_owner', 'public.agents', 'SELECT') + or not has_column_privilege('leoclean_kb_stage_owner', 'public.agents', 'id', 'SELECT') + or not has_column_privilege('leoclean_kb_stage_owner', 'public.agents', 'handle', 'SELECT') + or not has_table_privilege('leoclean_kb_stage_owner', 'kb_stage.kb_proposals', 'SELECT') + or has_table_privilege('leoclean_kb_stage_owner', 'kb_stage.kb_proposals', 'INSERT') then + raise exception 'leoclean_kb_stage_owner does not have the exact read/insert grant shape'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I', namespace.nspname, relation.relname, attribute.attname), + ', ' + order by namespace.nspname, relation.relname, attribute.attnum + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege('leoclean_kb_stage_owner', relation.oid, attribute.attnum, 'INSERT') + and not ( + namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' + and attribute.attname in ( + 'proposal_type', + 'status', + 'proposed_by_handle', + 'proposed_by_agent_id', + 'channel', + 'source_ref', + 'rationale', + 'payload' + ) + ); + if unexpected is not null then + raise exception 'leoclean_kb_stage_owner can INSERT unexpected columns: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name), + ', ' + order by namespace.nspname, relation.relname, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and has_table_privilege('leoclean_kb_stage_owner', relation.oid, privilege.name); + if unexpected is not null then + raise exception 'leoclean_kb_stage_owner unexpectedly has table-level DML: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I:%s', namespace.nspname, relation.relname, attribute.attname, privilege.name), + ', ' + order by namespace.nspname, relation.relname, attribute.attnum, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('UPDATE'), ('REFERENCES')) privilege(name) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege('leoclean_kb_stage_owner', relation.oid, attribute.attnum, privilege.name); + if unexpected is not null then + raise exception 'leoclean_kb_stage_owner unexpectedly has column-level DML: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I.%I', namespace.nspname, relation.relname, attribute.attname), + ', ' + order by namespace.nspname, relation.relname, attribute.attnum + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and has_column_privilege('leoclean_kb_stage_owner', relation.oid, attribute.attnum, 'SELECT') + and not ( + (namespace.nspname = 'kb_stage' and relation.relname = 'kb_proposals') + or ( + namespace.nspname = 'public' + and relation.relname = 'agents' + and attribute.attname in ('id', 'handle') + ) + ); + if unexpected is not null then + raise exception 'leoclean_kb_stage_owner can SELECT unexpected columns: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s:%s', namespace.nspname, relation.relname, scoped_role.name, privilege.name), + ', ' + order by namespace.nspname, relation.relname, scoped_role.name, privilege.name + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + cross join (values ('leoclean_kb_runtime'), ('leoclean_kb_stage_owner')) scoped_role(name) + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind = 'S' + and has_sequence_privilege(scoped_role.name, relation.oid, privilege.name); + if unexpected is not null then + raise exception 'a scoped Leo role unexpectedly has sequence privileges: %', unexpected; + end if; + + if exists ( + select 1 + from pg_catalog.pg_attribute attribute + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attname in ( + 'proposal_type', + 'status', + 'proposed_by_handle', + 'proposed_by_agent_id', + 'channel', + 'source_ref', + 'rationale', + 'payload' + ) + and not has_column_privilege( + 'leoclean_kb_stage_owner', + attribute.attrelid, + attribute.attnum, + 'INSERT' + ) + ) then + raise exception 'leoclean_kb_stage_owner is missing a required INSERT column grant'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I(%s)', namespace.nspname, function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid)), + ', ' + ) + into unexpected + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname in ('public', 'kb_stage') + and function_row.prosecdef + and function_row.oid <> stage_function + and has_function_privilege('leoclean_kb_runtime', function_row.oid, 'EXECUTE'); + if unexpected is not null then + raise exception 'leoclean_kb_runtime can execute unexpected SECURITY DEFINER functions: %', unexpected; + end if; + + if not exists ( + select 1 + from pg_catalog.pg_proc function_row + where function_row.oid = stage_function + and function_row.prosecdef + and function_row.proowner = owner_oid + and function_row.proconfig = array['search_path=pg_catalog, pg_temp']::text[] + ) then + raise exception 'stage_leoclean_proposal owner/security/search_path contract is invalid'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format( + 'grantee=%s privilege=%s grantable=%s', + acl.grantee, + acl.privilege_type, + acl.is_grantable + ), + ', ' + ) + into unexpected + from pg_catalog.pg_proc function_row + cross join lateral pg_catalog.aclexplode( + coalesce( + function_row.proacl, + pg_catalog.acldefault('f', function_row.proowner) + ) + ) as acl + where function_row.oid = stage_function + and not ( + acl.grantee in (runtime_oid, owner_oid) + and acl.privilege_type = 'EXECUTE' + and (acl.grantee = owner_oid or not acl.is_grantable) + ); + if unexpected is not null then + raise exception 'stage_leoclean_proposal has unexpected ACL entries: %', unexpected; + end if; + + if not has_function_privilege( + 'leoclean_kb_runtime', + 'kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)', + 'EXECUTE' + ) then + raise exception 'leoclean_kb_runtime cannot execute stage_leoclean_proposal'; + end if; +end +$verification$; + +-- LOGIN is the last state change in the same transaction as every database ACL, +-- canonical privilege, large-object, and topology assertion. Any failure rolls +-- back the complete privilege migration and leaves the role NOLOGIN. +alter role leoclean_kb_runtime + with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; + +do $login_verification$ +begin + if exists ( + select 1 + from pg_catalog.pg_roles role_row + where role_row.rolname = 'leoclean_kb_runtime' + and ( + not role_row.rolcanlogin + or role_row.rolsuper + or role_row.rolcreatedb + or role_row.rolcreaterole + or role_row.rolinherit + or role_row.rolreplication + or role_row.rolbypassrls + or role_row.rolconnlimit <> 8 + ) + ) then + raise exception 'leoclean_kb_runtime final login attributes are unsafe'; + end if; +end +$login_verification$; + +commit; diff --git a/ops/observatory_read_role.sql b/ops/observatory_read_role.sql index 34f036e..e7381be 100644 --- a/ops/observatory_read_role.sql +++ b/ops/observatory_read_role.sql @@ -161,6 +161,37 @@ begin raise exception 'Observatory principal can SELECT outside its allowlist: %', unexpected; end if; + select pg_catalog.string_agg(protected_schema.schema_name, ', ' order by protected_schema.schema_name) + into unexpected + from (values ('public'), ('kb_stage')) protected_schema(schema_name) + where pg_catalog.has_schema_privilege(principal, protected_schema.schema_name, 'CREATE'); + if unexpected is not null then + raise exception 'Observatory principal can create objects in protected schemas: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format( + '%I.%I(%s)', + namespace.nspname, + procedure.proname, + pg_catalog.oidvectortypes(procedure.proargtypes) + ), + ', ' order by procedure.proname, pg_catalog.oidvectortypes(procedure.proargtypes) + ) + into unexpected + from pg_catalog.pg_proc procedure + join pg_catalog.pg_namespace namespace on namespace.oid = procedure.pronamespace + where namespace.nspname = 'kb_stage' + and procedure.proname in ( + 'approve_strict_proposal', + 'assert_approved_proposal', + 'finish_approved_proposal' + ) + and pg_catalog.has_function_privilege(principal, procedure.oid, 'EXECUTE'); + if unexpected is not null then + raise exception 'Observatory principal can execute protected proposal/apply gates: %', unexpected; + end if; + if not ( pg_catalog.has_table_privilege(principal, 'public.claims', 'SELECT') and pg_catalog.has_table_privilege(principal, 'public.sources', 'SELECT') @@ -179,5 +210,7 @@ select jsonb_build_object( 'database_principal', current_setting('observatory.iam_user'), 'required_reads', true, 'effective_table_writes_denied', true, + 'effective_schema_create_denied', true, + 'protected_apply_execute_denied', true, 'default_transaction_read_only', true )::text; diff --git a/ops/provision_gcp_leoclean_runtime_role.sh b/ops/provision_gcp_leoclean_runtime_role.sh new file mode 100755 index 0000000..9ff89f8 --- /dev/null +++ b/ops/provision_gcp_leoclean_runtime_role.sh @@ -0,0 +1,150 @@ +#!/bin/bash +# Provision the scoped GCP staging role without placing its cleartext password +# in SQL text, argv, command history, or PostgreSQL statement logs. +set +x +set -euo pipefail +umask 077 +ulimit -c 0 + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PATH +runtime_password=${TELEO_LEOCLEAN_DB_PASSWORD-} +administrator_password=${PGPASSWORD-} +unset TELEO_LEOCLEAN_DB_PASSWORD PGPASSWORD + +script_path=${BASH_SOURCE[0]} +case "$script_path" in + */*) script_directory=${script_path%/*} ;; + *) script_directory=. ;; +esac +SCRIPT_DIR="$(cd -- "$script_directory" && pwd -P)" +SQL_FILE="$SCRIPT_DIR/gcp_leoclean_runtime_role.sql" +SERVER_CA_FILE="$SCRIPT_DIR/gcp-teleo-pgvector-standby-server-ca.pem" +PSQL_BIN=/usr/bin/psql +SETSID_BIN=/usr/bin/setsid +ENV_BIN=/usr/bin/env +SHA256SUM_BIN=/usr/bin/sha256sum +SERVER_CA_SHA256=80701e768f0e1f6b9d621aa0b53f6e851daaa276c6d9a8e51a300fbc015539cb + +assert_root_owned_nonwritable() { + local target="$1" state uid mode + state="$(stat -Lc '%u:%a' -- "$target")" + uid=${state%%:*} + mode=${state#*:} + if [ "$uid" != 0 ] || (( (8#$mode & 8#022) != 0 )); then + echo "Refusing an untrusted PostgreSQL executable path." >&2 + return 1 + fi +} + +assert_trusted_executable() { + local requested="$1" resolved current + [ -x "$requested" ] || { + echo "Required PostgreSQL client is unavailable." >&2 + return 1 + } + resolved="$(readlink -f -- "$requested")" + [ -n "$resolved" ] && [ -f "$resolved" ] && [ -x "$resolved" ] || { + echo "Required PostgreSQL client did not resolve to an executable file." >&2 + return 1 + } + assert_root_owned_nonwritable "$requested" + current="$(dirname "$requested")" + while :; do + assert_root_owned_nonwritable "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done + current="$resolved" + while :; do + assert_root_owned_nonwritable "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done +} + +if [ ! -f "$SQL_FILE" ] || [ -L "$SQL_FILE" ] || [ ! -f "$SERVER_CA_FILE" ] || [ -L "$SERVER_CA_FILE" ]; then + echo "Reviewed GCP runtime-role SQL is unavailable." >&2 + exit 66 +fi +assert_trusted_executable "$PSQL_BIN" +assert_trusted_executable "$SETSID_BIN" +assert_trusted_executable "$ENV_BIN" +assert_trusted_executable "$SHA256SUM_BIN" +if [ "$("$SHA256SUM_BIN" "$SERVER_CA_FILE")" != "$SERVER_CA_SHA256 $SERVER_CA_FILE" ]; then + echo "Reviewed Cloud SQL server CA does not match the pinned certificate." >&2 + exit 66 +fi + +if [ -z "$runtime_password" ] || [ "${#runtime_password}" -gt 4096 ]; then + echo "TELEO_LEOCLEAN_DB_PASSWORD must contain a bounded nonempty value." >&2 + exit 64 +fi +case "$runtime_password" in + *$'\n'*|*$'\r'*) + echo "TELEO_LEOCLEAN_DB_PASSWORD must not contain line breaks." >&2 + exit 64 + ;; +esac +if [ -z "$administrator_password" ] || [ "${#administrator_password}" -gt 4096 ]; then + echo "PGPASSWORD must contain a bounded nonempty administrator credential." >&2 + exit 64 +fi +case "$administrator_password" in + *$'\n'*|*$'\r'*) + echo "PGPASSWORD must not contain line breaks." >&2 + exit 64 + ;; +esac +cleanup_password() { + runtime_password= + administrator_password= + unset runtime_password administrator_password +} +provision_pid= +terminate_provisioning() { + local exit_status="$1" + trap - INT TERM + if [[ "$provision_pid" =~ ^[1-9][0-9]*$ ]]; then + kill -TERM -- "-$provision_pid" 2>/dev/null || kill -TERM -- "$provision_pid" 2>/dev/null || true + wait "$provision_pid" 2>/dev/null || true + provision_pid= + fi + cleanup_password + exit "$exit_status" +} +trap cleanup_password EXIT +trap 'terminate_provisioning 130' INT +trap 'terminate_provisioning 143' TERM + +# \password encrypts client-side. --no-password ensures an administrator-login +# prompt cannot consume either of the two runtime-password records from stdin. +# setsid detaches psql from any controlling terminal without --fork, keeping +# its PID as the session/process-group leader so an interruption can terminate +# and reap the complete credential-bearing child before this wrapper returns. +{ + printf '%s\n' "$runtime_password" + printf '%s\n' "$runtime_password" +} | "$ENV_BIN" -i \ + PATH="$PATH" \ + LANG=C LC_ALL=C \ + PYTHONNOUSERSITE=1 PYTHONSAFEPATH=1 \ + PGHOST=10.61.0.3 \ + PGPORT=5432 \ + PGDATABASE=teleo_canonical \ + PGUSER=postgres \ + PGPASSWORD="$administrator_password" \ + PGSSLMODE=verify-ca \ + PGSSLROOTCERT="$SERVER_CA_FILE" \ + "$SETSID_BIN" -- "$PSQL_BIN" \ + --no-psqlrc \ + --no-password \ + --set=ON_ERROR_STOP=1 \ + --file="$SQL_FILE" & +provision_pid=$! +set +e +wait "$provision_pid" +status=$? +set -e +provision_pid= +exit "$status" diff --git a/ops/verify_gcp_leoclean_runtime_permissions.py b/ops/verify_gcp_leoclean_runtime_permissions.py new file mode 100644 index 0000000..09038a9 --- /dev/null +++ b/ops/verify_gcp_leoclean_runtime_permissions.py @@ -0,0 +1,2233 @@ +#!/usr/bin/python3 -I +"""Retain a sanitized least-privilege receipt for the GCP Leo runtime role. + +This verifier is intentionally service-independent. It must run on the private +GCP VM as ``teleo`` and connects directly to the private Cloud SQL address. The +only database credential is read from the scoped Secret Manager secret into +process memory, passed to each ``psql`` child through ``PGPASSWORD``, and never +included in an argument, message, receipt, or output file. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pwd +import re +import stat +import subprocess +import sys +from collections.abc import Callable, Mapping +from contextlib import nullcontext +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +PROJECT_ID = "teleo-501523" +PRIVATE_CLOUDSQL_HOST = "10.61.0.3" +PRIVATE_CLOUDSQL_PORT = 5432 +EXPECTED_SYSTEM_IDENTIFIER = "7659718422914359312" +CANONICAL_DATABASE = "teleo_canonical" +LEGACY_DATABASE = "teleo_kb" +TEMPLATE_DATABASE = "template1" +RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime" +STAGE_OWNER_DATABASE_ROLE = "leoclean_kb_stage_owner" +RUNTIME_UNIX_USER = "teleo" +SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" +ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password" +GCLOUD_BIN = "/usr/bin/gcloud" +PSQL_BIN = "/usr/bin/psql" +SERVER_CA_PATH = Path("/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem") +RUNTIME_CLOUDSDK_CONFIG = Path("/usr/local/libexec/livingip/leoclean-kb/gcloud-config") +SERVER_CA_SHA256 = "80701e768f0e1f6b9d621aa0b53f6e851daaa276c6d9a8e51a300fbc015539cb" +COMMAND_TIMEOUT_SECONDS = 30 +CHILD_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +RUN_ID_RE = re.compile(r"[a-z0-9][a-z0-9-]{6,62}[a-z0-9]\Z") +SQLSTATE_RE = re.compile(r"(?m)^(?:ERROR|FATAL):\s+([0-9A-Z]{5}):") +IAM_PERMISSION = "secretmanager.versions.access" +REQUIRED_TIER = "T3_live_readonly" +LOWER_TIER = "T2_runtime" +ARTIFACT_NAME = "gcp_leoclean_runtime_permissions" +STAGE_FUNCTION_SIGNATURE = "kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)" + +# PostgreSQL stores the exact dollar-quoted function body in pg_proc.prosrc. +# A source-parity regression below binds this constant to the provisioning SQL, +# while the live verifier compares the catalog body without retaining it. +EXPECTED_STAGE_FUNCTION_SOURCE = """ +declare + staged kb_stage.kb_proposals%rowtype; + proposer_id uuid; +begin + if session_user <> 'leoclean_kb_runtime' then + raise exception 'stage_leoclean_proposal is restricted to leoclean_kb_runtime' + using errcode = '42501'; + end if; + if p_proposal_type not in ('revise_claim', 'revise_strategy', 'add_edge') then + raise exception 'unsupported Leo proposal type: %', p_proposal_type using errcode = '22023'; + end if; + if nullif(btrim(p_source_ref), '') is null then + raise exception 'source_ref is required' using errcode = '22023'; + end if; + if nullif(btrim(p_rationale), '') is null then + raise exception 'rationale is required' using errcode = '22023'; + end if; + if p_payload is null or jsonb_typeof(p_payload) <> 'object' then + raise exception 'proposal payload must be a JSON object' using errcode = '22023'; + end if; + + select agent.id + into proposer_id + from public.agents as agent + where agent.handle = 'leo'; + + if proposer_id is null then + raise exception 'canonical Leo agent row is required' using errcode = '23503'; + end if; + + insert into kb_stage.kb_proposals ( + proposal_type, + status, + proposed_by_handle, + proposed_by_agent_id, + channel, + source_ref, + rationale, + payload + ) values ( + p_proposal_type, + 'pending_review', + 'leo', + proposer_id, + coalesce(nullif(p_channel, ''), 'telegram'), + p_source_ref, + p_rationale, + p_payload + ) + returning * into staged; + + return to_jsonb(staged); +end +""" +EXPECTED_STAGE_FUNCTION_SOURCE_SHA256 = "89f78d5ad7bc7227a4588baf42f7f77e2162b3c4647a250f7c88716b0debfdea" +MAX_STAGE_FUNCTION_SOURCE_BYTES = 65_536 + +STAGE_FUNCTION_METADATA_EXPECTATIONS: tuple[tuple[str, Any], ...] = ( + ("argument_defaults", 0), + ( + "argument_names", + ["p_proposal_type", "p_channel", "p_source_ref", "p_rationale", "p_payload"], + ), + ("argument_types", ["text", "text", "text", "text", "jsonb"]), + ("kind", "f"), + ("language", "plpgsql"), + ("leakproof", False), + ("owner", STAGE_OWNER_DATABASE_ROLE), + ("parallel", "u"), + ("returns_set", False), + ("result", "jsonb"), + ("security_definer", True), + ("strict", False), + ("volatility", "v"), +) + +STAGE_OWNER_INSERT_COLUMNS: tuple[str, ...] = ( + "proposal_type", + "status", + "proposed_by_handle", + "proposed_by_agent_id", + "channel", + "source_ref", + "rationale", + "payload", +) + +FUNCTION_PRIVILEGE_EXPECTATIONS: tuple[tuple[str, str, bool, bool], ...] = ( + ( + "stage_leoclean_proposal_5_arg", + "kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)", + True, + True, + ), + ( + "stage_leoclean_proposal_6_arg", + "kb_stage.stage_leoclean_proposal(text,text,text,text,text,jsonb)", + False, + False, + ), + ( + "approve_strict_proposal", + "kb_stage.approve_strict_proposal(uuid,text,jsonb,text,text)", + True, + False, + ), + ( + "assert_approved_proposal", + "kb_stage.assert_approved_proposal(uuid,text,jsonb,text,uuid,timestamptz,text)", + True, + False, + ), + ( + "finish_approved_proposal", + "kb_stage.finish_approved_proposal(uuid,text,jsonb,text,uuid,timestamptz,text,text)", + True, + False, + ), +) + +READ_TABLE_ALLOWLIST: tuple[tuple[str, str], ...] = ( + ("public", "agents"), + ("public", "claims"), + ("public", "claim_evidence"), + ("public", "claim_edges"), + ("public", "sources"), + ("public", "personas"), + ("public", "strategies"), + ("public", "beliefs"), + ("public", "blindspots"), + ("public", "agent_roles"), + ("public", "peer_models"), + ("public", "behavioral_rules"), + ("public", "contributor_rules"), + ("public", "reasoning_tools"), + ("public", "governance_gates"), + ("kb_stage", "kb_proposals"), +) + +CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( + "column_dml_grants", + "database_create_grants", + "missing_allowed_table_selects", + "large_object_acl_privileges", + "large_object_mutation_routine_execute", + "public_large_object_mutation_routine_execute", + "other_scoped_backends", + "owned_database_objects", + "owned_large_objects", + "owner_database_create_grants", + "owner_database_connect_grants", + "owner_direct_database_acl_entries", + "owner_missing_agent_select_columns", + "owner_missing_proposal_insert_columns", + "owner_schema_create_grants", + "owner_sequence_grants", + "owner_table_dml_grants", + "owner_unexpected_column_dml_grants", + "owner_unexpected_column_selects", + "owner_unexpected_owned_database_objects", + "owner_unexpected_routine_execute", + "owner_unexpected_schema_usage", + "owner_unexpected_table_selects", + "schema_create_grants", + "parameter_privileges", + "public_database_connect_grants", + "scoped_prepared_xacts", + "sequence_grants", + "table_dml_grants", + "unexpected_column_selects", + "unexpected_direct_database_acl_entries", + "unexpected_connectable_databases", + "unexpected_routine_execute", + "unexpected_role_settings", + "unexpected_schema_usage", + "unexpected_security_definer_execute", + "unexpected_table_selects", + "unsafe_allowed_relation_kinds", + "unsafe_allowed_relation_inheritance", + "unsafe_proposal_column_contract", + "unsafe_proposal_default_dependencies", + "unsafe_proposal_defaults", + "unsafe_proposal_constraint_dependencies", + "unsafe_proposal_index_expressions", + "unsafe_proposal_policies", + "unsafe_proposal_rewrite_rules", + "unsafe_proposal_triggers", + "unsafe_proposal_generated_columns", +) + +CATALOG_TRUE_FIELDS: tuple[str, ...] = ( + "canonical_connect", + "owner_grant_contract", + "owner_schema_usage", + "proposal_table_contract", + "proposal_constraint_shape", + "runtime_connect_acl_exact", + "runtime_schema_usage", + "runtime_setting_contract", + "effective_setting_contract", +) + + +@dataclass(frozen=True) +class CommandResult: + returncode: int + stdout: str + stderr: str + + +Runner = Callable[[list[str], Mapping[str, str], int, bool], CommandResult] +DependencyValidator = Callable[[], None] + + +@dataclass(frozen=True) +class NegativeCheck: + name: str + database: str + sql: str + expected_sqlstate: str + expected_connection_denial: bool = False + + +class VerificationError(RuntimeError): + """A deliberately sanitized, machine-readable verification failure.""" + + def __init__( + self, + code: str, + check: str, + *, + expected_sqlstate: str | None = None, + observed_sqlstate: str | None = None, + ) -> None: + self.code = code + self.check = check + self.expected_sqlstate = expected_sqlstate + self.observed_sqlstate = observed_sqlstate + super().__init__(f"{code}:{check}") + + def as_dict(self) -> dict[str, str]: + payload = {"check": self.check, "code": self.code} + if self.expected_sqlstate is not None: + payload["expected_sqlstate"] = self.expected_sqlstate + if self.observed_sqlstate is not None: + payload["observed_sqlstate"] = self.observed_sqlstate + return payload + + +def subprocess_runner( + command: list[str], + env: Mapping[str, str], + timeout: int, + discard_stdout: bool, +) -> CommandResult: + completed = subprocess.run( + command, + check=False, + text=True, + env=dict(env), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL if discard_stdout else subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + return CommandResult( + returncode=completed.returncode, + stdout="" if discard_stdout else (completed.stdout or ""), + stderr=completed.stderr or "", + ) + + +def validate_run_id(value: str) -> str: + if not RUN_ID_RE.fullmatch(value): + raise ValueError("run id must be 8-64 lowercase letters, digits, or hyphens") + return value + + +def _run_id_arg(value: str) -> str: + try: + return validate_run_id(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(str(exc)) from None + + +def _base_child_environment(_source: Mapping[str, str]) -> dict[str, str]: + return { + "HOME": "/home/teleo", + "LANG": "C", + "LC_ALL": "C", + "PATH": CHILD_PATH, + "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", + } + + +def _path_and_parents(path: Path) -> tuple[Path, ...]: + resolved = path if path.is_absolute() else path.absolute() + return (resolved, *resolved.parents) + + +def _assert_root_owned_nonwritable(path: Path, *, follow_symlinks: bool = True) -> None: + try: + metadata = path.stat() if follow_symlinks else path.lstat() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if metadata.st_uid != 0 or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + if os.geteuid() != 0 and os.access(path, os.W_OK): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + + +def _assert_trusted_executable(raw_path: str) -> None: + requested = Path(raw_path) + if not requested.is_absolute(): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + try: + resolved = requested.resolve(strict=True) + metadata = resolved.stat() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if not stat.S_ISREG(metadata.st_mode) or not os.access(resolved, os.X_OK): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + try: + requested_metadata = requested.lstat() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if requested_metadata.st_uid != 0 or ( + not stat.S_ISLNK(requested_metadata.st_mode) and requested_metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + ): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + for candidate in (*_path_and_parents(requested.parent), *_path_and_parents(resolved)): + _assert_root_owned_nonwritable(candidate) + + +def validate_runtime_dependencies() -> None: + _assert_trusted_executable(GCLOUD_BIN) + _assert_trusted_executable(PSQL_BIN) + runtime_cloudsdk_config_path() + server_ca = runtime_server_ca_path() + for candidate in _path_and_parents(server_ca): + _assert_root_owned_nonwritable(candidate) + try: + certificate = server_ca.read_bytes() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if hashlib.sha256(certificate).hexdigest() != SERVER_CA_SHA256: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + + +def runtime_server_ca_path() -> Path: + raw = os.environ.get("TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT", str(SERVER_CA_PATH)) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if resolved != candidate or not resolved.is_file(): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + return resolved + + +def selected_server_ca_path() -> Path: + return Path(os.environ.get("TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT", str(SERVER_CA_PATH))) + + +def runtime_cloudsdk_config_path() -> Path: + raw = os.environ.get("TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG", str(RUNTIME_CLOUDSDK_CONFIG)) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + if resolved != candidate or not resolved.is_dir() or any(resolved.iterdir()): + raise OSError + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + for part in _path_and_parents(resolved): + _assert_root_owned_nonwritable(part) + return resolved + + +def selected_cloudsdk_config_path() -> Path: + return Path(os.environ.get("TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG", str(RUNTIME_CLOUDSDK_CONFIG))) + + +def _gcloud_environment(source: Mapping[str, str], config_dir: Path) -> dict[str, str]: + env = _base_child_environment(source) + env.update( + { + "CLOUDSDK_CONFIG": str(config_dir), + "CLOUDSDK_CORE_DISABLE_FILE_LOGGING": "true", + "CLOUDSDK_CORE_DISABLE_PROMPTS": "1", + "CLOUDSDK_CORE_PROJECT": PROJECT_ID, + } + ) + return env + + +def _psql_environment(source: Mapping[str, str], password: str) -> dict[str, str]: + env = _base_child_environment(source) + env.update( + { + "PGPASSWORD": password, + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": str(selected_server_ca_path()), + } + ) + return env + + +def _call( + command: list[str], + *, + env: Mapping[str, str], + runner: Runner, + check: str, + discard_stdout: bool = False, +) -> CommandResult: + try: + return runner(command, env, COMMAND_TIMEOUT_SECONDS, discard_stdout) + except FileNotFoundError: + raise VerificationError("command_not_found", check) from None + except subprocess.TimeoutExpired: + raise VerificationError("command_timed_out", check) from None + except OSError: + raise VerificationError("command_os_error", check) from None + + +def _secret_command(secret_name: str) -> list[str]: + return [ + GCLOUD_BIN, + "secrets", + "versions", + "access", + "latest", + f"--secret={secret_name}", + f"--project={PROJECT_ID}", + "--quiet", + ] + + +def _read_scoped_password(*, env: Mapping[str, str], runner: Runner) -> str: + result = _call( + _secret_command(SCOPED_PASSWORD_SECRET), + env=env, + runner=runner, + check="scoped_secret_access", + ) + if result.returncode != 0: + raise VerificationError("scoped_secret_access_failed", "scoped_secret_access") + + password = result.stdout[:-1] if result.stdout.endswith("\n") else result.stdout + if password.endswith("\r"): + password = password[:-1] + if not password or len(password) > 4096 or any(character in password for character in ("\x00", "\r", "\n")): + raise VerificationError("scoped_secret_value_invalid", "scoped_secret_access") + return password + + +def _assert_administrator_secret_denied(*, env: Mapping[str, str], runner: Runner) -> dict[str, Any]: + result = _call( + _secret_command(ADMINISTRATOR_PASSWORD_SECRET), + env=env, + runner=runner, + check="administrator_secret_access", + discard_stdout=True, + ) + if result.returncode == 0: + raise VerificationError("administrator_secret_unexpectedly_readable", "administrator_secret_access") + + permission_denied = re.search(r"\bPERMISSION_DENIED\b", result.stderr, re.IGNORECASE) is not None + exact_permission = IAM_PERMISSION in result.stderr.lower() + if not permission_denied or not exact_permission: + raise VerificationError("administrator_secret_denial_not_iam_permission", "administrator_secret_access") + return { + "classification": "iam_permission_denied", + "permission": IAM_PERMISSION, + "result": "denied", + "stdout_discarded": True, + } + + +def _conninfo(database: str) -> str: + return ( + f"host={PRIVATE_CLOUDSQL_HOST} port={PRIVATE_CLOUDSQL_PORT} dbname={database} " + f"user={RUNTIME_DATABASE_ROLE} sslmode=verify-ca sslrootcert={selected_server_ca_path()} connect_timeout=10 " + "application_name=leoclean_runtime_permission_verifier" + ) + + +def _psql_command(database: str, sql: str) -> list[str]: + return [ + PSQL_BIN, + "--no-psqlrc", + "--no-password", + "--quiet", + "--tuples-only", + "--no-align", + "--set=ON_ERROR_STOP=1", + "--set=VERBOSITY=verbose", + f"--dbname={_conninfo(database)}", + f"--command={sql}", + ] + + +def _single_output_line(result: CommandResult, check: str) -> str: + lines = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if len(lines) != 1: + raise VerificationError("unexpected_psql_output_shape", check) + return lines[0] + + +def _run_psql_success( + *, + check: str, + database: str, + sql: str, + env: Mapping[str, str], + runner: Runner, +) -> str: + result = _call( + _psql_command(database, sql), + env=env, + runner=runner, + check=check, + ) + if result.returncode != 0: + observed = sorted(set(SQLSTATE_RE.findall(result.stderr))) + raise VerificationError( + "psql_check_failed", + check, + observed_sqlstate=",".join(observed) if observed else "missing", + ) + return _single_output_line(result, check) + + +def _parse_json_object(raw: str, check: str) -> dict[str, Any]: + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + raise VerificationError("psql_json_invalid", check) from None + if not isinstance(parsed, dict): + raise VerificationError("psql_json_not_object", check) + return parsed + + +def _parse_zero_count(raw: str, check: str) -> int: + if raw != "0": + raise VerificationError("canary_row_count_nonzero", check) + return 0 + + +def _sql_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def normalize_stage_function_source(source: str) -> str: + normalized = source.replace("\r\n", "\n").replace("\r", "\n") + return normalized.strip("\n") + "\n" + + +def _identity_sql() -> str: + return """ +select pg_catalog.jsonb_build_object( + 'database', pg_catalog.current_database(), + 'current_user', current_user, + 'session_user', session_user, + 'server_addr', pg_catalog.inet_server_addr()::text, + 'server_port', pg_catalog.inet_server_port(), + 'ssl', coalesce( + (select ssl from pg_catalog.pg_stat_ssl where pid = pg_catalog.pg_backend_pid()), + false + ), + 'ssl_version', coalesce( + (select version from pg_catalog.pg_stat_ssl where pid = pg_catalog.pg_backend_pid()), + '' + ), + 'system_identifier', (select system_identifier::text from pg_catalog.pg_control_system()) +)::text; +""".strip() + + +def _database_role_posture_sql(role_name: str) -> str: + return f""" +select pg_catalog.jsonb_build_object( + 'role', role_row.rolname, + 'can_login', role_row.rolcanlogin, + 'is_superuser', role_row.rolsuper, + 'can_create_db', role_row.rolcreatedb, + 'can_create_role', role_row.rolcreaterole, + 'inherits_privileges', role_row.rolinherit, + 'can_replicate', role_row.rolreplication, + 'bypasses_rls', role_row.rolbypassrls, + 'connection_limit', role_row.rolconnlimit, + 'direct_membership_edges', ( + select pg_catalog.count(*) + from pg_catalog.pg_auth_members membership + where membership.roleid = role_row.oid + or membership.member = role_row.oid + ) +)::text +from pg_catalog.pg_roles role_row +where role_row.rolname = {_sql_literal(role_name)}; +""".strip() + + +def _role_posture_sql() -> str: + return _database_role_posture_sql(RUNTIME_DATABASE_ROLE) + + +def _stage_owner_role_posture_sql() -> str: + return _database_role_posture_sql(STAGE_OWNER_DATABASE_ROLE) + + +def _function_privilege_posture_sql() -> str: + values_sql = ",\n ".join( + f"({_sql_literal(name)}, {_sql_literal(signature)})" + for name, signature, _expected_exists, _expected_execute in FUNCTION_PRIVILEGE_EXPECTATIONS + ) + return f""" +with expected(check_name, signature) as ( + values + {values_sql} +), resolved as ( + select check_name, + signature, + pg_catalog.to_regprocedure(signature) as function_oid + from expected +) +select pg_catalog.jsonb_object_agg( + check_name, + pg_catalog.jsonb_build_object( + 'exists', function_oid is not null, + 'execute', coalesce( + pg_catalog.has_function_privilege(current_user, function_oid::pg_catalog.oid, 'EXECUTE'), + false + ) + ) +)::text +from resolved; +""".strip() + + +def _stage_function_definition_sql() -> str: + return f""" +with target as ( + select pg_catalog.to_regprocedure({_sql_literal(STAGE_FUNCTION_SIGNATURE)})::pg_catalog.oid as oid +), function_row as ( + select target.oid, + function_catalog.proowner, + function_catalog.prolang, + function_catalog.prokind, + function_catalog.prosecdef, + function_catalog.proleakproof, + function_catalog.proisstrict, + function_catalog.proretset, + function_catalog.provolatile, + function_catalog.proparallel, + function_catalog.pronargdefaults, + function_catalog.proargnames, + function_catalog.proargmodes, + function_catalog.proargtypes, + function_catalog.proconfig, + function_catalog.proacl, + function_catalog.prosrc, + owner_role.rolname as owner_name, + language_row.lanname as language_name + from target + left join pg_catalog.pg_proc function_catalog on function_catalog.oid = target.oid + left join pg_catalog.pg_roles owner_role on owner_role.oid = function_catalog.proowner + left join pg_catalog.pg_language language_row on language_row.oid = function_catalog.prolang +), runtime_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(RUNTIME_DATABASE_ROLE)} +), owner_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(STAGE_OWNER_DATABASE_ROLE)} +) +select pg_catalog.jsonb_build_object( + 'exists', function_row.oid is not null, + 'owner', function_row.owner_name, + 'language', function_row.language_name, + 'kind', function_row.prokind, + 'security_definer', function_row.prosecdef, + 'leakproof', function_row.proleakproof, + 'strict', function_row.proisstrict, + 'returns_set', function_row.proretset, + 'volatility', function_row.provolatile, + 'parallel', function_row.proparallel, + 'argument_defaults', function_row.pronargdefaults, + 'argument_names', function_row.proargnames, + 'argument_modes', function_row.proargmodes, + 'argument_types', ( + select pg_catalog.jsonb_agg(pg_catalog.format_type(argument_type, null) order by ordinal) + from pg_catalog.unnest(function_row.proargtypes::pg_catalog.oid[]) + with ordinality argument(argument_type, ordinal) + ), + 'result', pg_catalog.pg_get_function_result(function_row.oid), + 'configuration', function_row.proconfig, + 'runtime_execute', coalesce( + pg_catalog.has_function_privilege(current_user, function_row.oid, 'EXECUTE'), + false + ), + 'acl_exact', coalesce(( + select pg_catalog.count(*) filter ( + where acl.grantee = runtime_role.oid + and acl.privilege_type = 'EXECUTE' + and not acl.is_grantable + ) = 1 + and pg_catalog.count(*) filter ( + where not ( + acl.grantee in (runtime_role.oid, owner_role.oid) + and acl.privilege_type = 'EXECUTE' + and (acl.grantee = owner_role.oid or not acl.is_grantable) + ) + ) = 0 + from runtime_role + cross join owner_role + cross join lateral pg_catalog.aclexplode( + coalesce(function_row.proacl, pg_catalog.acldefault('f', function_row.proowner)) + ) acl + ), false), + 'source', function_row.prosrc +)::text +from function_row; +""".strip() + + +def _catalog_privilege_posture_sql() -> str: + allowlist_values = ",\n ".join( + f"({_sql_literal(schema)}, {_sql_literal(relation)})" for schema, relation in READ_TABLE_ALLOWLIST + ) + owner_insert_values = ",\n ".join(f"({_sql_literal(column_name)})" for column_name in STAGE_OWNER_INSERT_COLUMNS) + return f""" +with runtime_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(RUNTIME_DATABASE_ROLE)} +), owner_role as ( + select oid from pg_catalog.pg_roles where rolname = {_sql_literal(STAGE_OWNER_DATABASE_ROLE)} +), stage_function as ( + select pg_catalog.to_regprocedure({_sql_literal(STAGE_FUNCTION_SIGNATURE)})::pg_catalog.oid as oid +), app_schemas as ( + select oid, nspname + from pg_catalog.pg_namespace + where nspname <> 'information_schema' + and nspname !~ '^pg_' +), allowed_select(nspname, relname) as ( + values + {allowlist_values} +), allowed_relation as ( + select allowed_select.nspname, + allowed_select.relname, + relation.oid, + relation.relkind + from allowed_select + left join pg_catalog.pg_namespace namespace + on namespace.nspname = allowed_select.nspname + left join pg_catalog.pg_class relation + on relation.relnamespace = namespace.oid + and relation.relname = allowed_select.relname +), owner_insert_column(attname) as ( + values + {owner_insert_values} +), expected_proposal_column(ordinal, attname, typname, not_null, has_default, collated) as ( + values + (1, 'id', 'uuid', true, true, false), + (2, 'proposal_type', 'text', true, false, true), + (3, 'status', 'text', true, true, true), + (4, 'proposed_by_handle', 'text', false, false, true), + (5, 'proposed_by_agent_id', 'uuid', false, false, false), + (6, 'channel', 'text', true, true, true), + (7, 'source_ref', 'text', false, false, true), + (8, 'rationale', 'text', true, false, true), + (9, 'payload', 'jsonb', true, false, false), + (10, 'reviewed_by_handle', 'text', false, false, true), + (11, 'reviewed_by_agent_id', 'uuid', false, false, false), + (12, 'reviewed_at', 'timestamptz', false, false, false), + (13, 'review_note', 'text', false, false, true), + (14, 'applied_by_handle', 'text', false, false, true), + (15, 'applied_by_agent_id', 'uuid', false, false, false), + (16, 'applied_at', 'timestamptz', false, false, false), + (17, 'created_at', 'timestamptz', true, true, false), + (18, 'updated_at', 'timestamptz', true, true, false) +), proposal_table as ( + select relation.* + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' +), expected_proposal_default(attname, expression) as ( + values + ('id', 'gen_random_uuid()'), + ('status', '''pending_review''::text'), + ('channel', '''cli''::text'), + ('created_at', 'now()'), + ('updated_at', 'now()') +), actual_proposal_default as ( + select attribute.attname, + pg_catalog.pg_get_expr(default_row.adbin, default_row.adrelid) as expression + from proposal_table + join pg_catalog.pg_attribute attribute on attribute.attrelid = proposal_table.oid + join pg_catalog.pg_attrdef default_row + on default_row.adrelid = attribute.attrelid + and default_row.adnum = attribute.attnum + where attribute.attnum > 0 + and not attribute.attisdropped +) +select pg_catalog.jsonb_build_object( + 'canonical_connect', coalesce( + pg_catalog.has_database_privilege(current_user, pg_catalog.current_database(), 'CONNECT'), + false + ), + 'runtime_connect_acl_exact', coalesce(( + select pg_catalog.count(*) filter ( + where database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + and acl.privilege_type = 'CONNECT' + and not acl.is_grantable + ) = 1 + from pg_catalog.pg_database database_row + cross join runtime_role + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) + and acl.grantee = runtime_role.oid + ), false), + 'runtime_setting_contract', coalesce(( + select pg_catalog.count(*) = 1 + and pg_catalog.bool_and( + setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ) + from pg_catalog.pg_db_role_setting setting_row + join runtime_role on runtime_role.oid = setting_row.setrole + ), false), + 'unexpected_role_settings', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_db_role_setting setting_row + where setting_row.setrole in ((select oid from runtime_role), (select oid from owner_role)) + and not ( + setting_row.setrole = (select oid from runtime_role) + and setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ) + ), + 'effective_setting_contract', ( + pg_catalog.current_setting('search_path') = 'pg_catalog, public, kb_stage' + and pg_catalog.current_setting('statement_timeout') = '15s' + and pg_catalog.current_setting('lock_timeout') = '2s' + and pg_catalog.current_setting('session_preload_libraries') = '' + and pg_catalog.current_setting('local_preload_libraries') = '' + and pg_catalog.current_setting('role') = 'none' + ), + 'unexpected_connectable_databases', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + where database_row.datname <> {_sql_literal(CANONICAL_DATABASE)} + and pg_catalog.has_database_privilege(current_user, database_row.oid, 'CONNECT') + ), + 'owner_database_connect_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + where pg_catalog.has_database_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + database_row.oid, + 'CONNECT' + ) + ), + 'public_database_connect_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where acl.grantee = 0 + and acl.privilege_type = 'CONNECT' + ), + 'unexpected_direct_database_acl_entries', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + cross join runtime_role + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where acl.grantee = runtime_role.oid + and not ( + database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + and acl.privilege_type = 'CONNECT' + and not acl.is_grantable + ) + ), + 'owner_direct_database_acl_entries', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + cross join owner_role + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where acl.grantee = owner_role.oid + ), + 'database_create_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + where pg_catalog.has_database_privilege(current_user, database_row.oid, 'CREATE') + ), + 'owner_database_create_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + where pg_catalog.has_database_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + database_row.oid, + 'CREATE' + ) + ), + 'owned_database_objects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_shdepend dependency + join runtime_role on runtime_role.oid = dependency.refobjid + where dependency.refclassid = 'pg_catalog.pg_authid'::pg_catalog.regclass + and dependency.deptype = 'o' + ), + 'owner_unexpected_owned_database_objects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_shdepend dependency + join owner_role on owner_role.oid = dependency.refobjid + cross join stage_function + where dependency.refclassid = 'pg_catalog.pg_authid'::pg_catalog.regclass + and dependency.deptype = 'o' + and not ( + dependency.dbid = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = pg_catalog.current_database() + ) + and dependency.classid = 'pg_catalog.pg_proc'::pg_catalog.regclass + and dependency.objid = stage_function.oid + and dependency.objsubid = 0 + ) + ), + 'runtime_schema_usage', coalesce( + pg_catalog.has_schema_privilege(current_user, 'public', 'USAGE') + and pg_catalog.has_schema_privilege(current_user, 'kb_stage', 'USAGE'), + false + ), + 'owner_schema_usage', coalesce( + pg_catalog.has_schema_privilege({_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, 'public', 'USAGE') + and pg_catalog.has_schema_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + 'kb_stage', + 'USAGE' + ), + false + ), + 'unexpected_schema_usage', ( + select pg_catalog.count(*)::int + from app_schemas + where app_schemas.nspname not in ('public', 'kb_stage') + and pg_catalog.has_schema_privilege(current_user, app_schemas.oid, 'USAGE') + ), + 'owner_unexpected_schema_usage', ( + select pg_catalog.count(*)::int + from app_schemas + where app_schemas.nspname not in ('public', 'kb_stage') + and pg_catalog.has_schema_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + app_schemas.oid, + 'USAGE' + ) + ), + 'schema_create_grants', ( + select pg_catalog.count(*)::int + from app_schemas + where pg_catalog.has_schema_privilege(current_user, app_schemas.oid, 'CREATE') + ), + 'owner_schema_create_grants', ( + select pg_catalog.count(*)::int + from app_schemas + where pg_catalog.has_schema_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + app_schemas.oid, + 'CREATE' + ) + ), + 'missing_allowed_table_selects', ( + select pg_catalog.count(*)::int + from allowed_relation + where allowed_relation.oid is null + or not coalesce( + pg_catalog.has_table_privilege(current_user, allowed_relation.oid, 'SELECT'), + false + ) + ), + 'unsafe_allowed_relation_kinds', ( + select pg_catalog.count(*)::int + from allowed_relation + where allowed_relation.relkind is distinct from 'r'::"char" + ), + 'unsafe_allowed_relation_inheritance', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_inherits inheritance + where inheritance.inhparent in (select oid from allowed_relation where oid is not null) + or inheritance.inhrelid in (select oid from allowed_relation where oid is not null) + ), + 'unsafe_proposal_column_contract', ( + select pg_catalog.count(*)::int + from expected_proposal_column expected + full join ( + select attribute.attnum::int as ordinal, + attribute.attname, + type_row.typname, + attribute.attnotnull as not_null, + attribute.atthasdef as has_default, + attribute.attidentity, + attribute.attgenerated, + attribute.atttypmod, + type_namespace.nspname as type_namespace, + attribute.attcollation = 'pg_catalog.default'::pg_catalog.regcollation as collated + from proposal_table + join pg_catalog.pg_attribute attribute on attribute.attrelid = proposal_table.oid + join pg_catalog.pg_type type_row on type_row.oid = attribute.atttypid + join pg_catalog.pg_namespace type_namespace on type_namespace.oid = type_row.typnamespace + where attribute.attnum > 0 + and not attribute.attisdropped + ) actual using (ordinal, attname) + where expected.ordinal is null + or actual.ordinal is null + or expected.typname is distinct from actual.typname + or expected.not_null is distinct from actual.not_null + or expected.has_default is distinct from actual.has_default + or expected.collated is distinct from actual.collated + or actual.attidentity <> '' + or actual.attgenerated <> '' + or actual.atttypmod <> -1 + or actual.type_namespace <> 'pg_catalog' + ), + 'proposal_table_contract', coalesce(( + select pg_catalog.count(*) = 1 + and pg_catalog.bool_and( + proposal_table.relkind = 'r' + and proposal_table.relpersistence = 'p' + and not proposal_table.relrowsecurity + and not proposal_table.relforcerowsecurity + ) + from proposal_table + ), false), + 'unsafe_proposal_triggers', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_trigger trigger_row + join proposal_table on proposal_table.oid = trigger_row.tgrelid + where not trigger_row.tgisinternal + ), + 'unsafe_proposal_rewrite_rules', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_rewrite rewrite_row + join proposal_table on proposal_table.oid = rewrite_row.ev_class + ), + 'unsafe_proposal_policies', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_policy policy_row + join proposal_table on proposal_table.oid = policy_row.polrelid + ), + 'unsafe_proposal_generated_columns', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_attribute attribute + join proposal_table on proposal_table.oid = attribute.attrelid + where attribute.attnum > 0 + and not attribute.attisdropped + and attribute.attgenerated <> '' + ), + 'unsafe_proposal_defaults', ( + select pg_catalog.count(*)::int + from expected_proposal_default expected + full join actual_proposal_default actual using (attname) + where expected.expression is distinct from actual.expression + ), + 'unsafe_proposal_default_dependencies', ( + select pg_catalog.count(*)::int + from ( + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + join pg_catalog.pg_proc function_row on function_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname <> 'pg_catalog' + or function_row.oid not in ( + 'pg_catalog.gen_random_uuid()'::pg_catalog.regprocedure, + 'pg_catalog.now()'::pg_catalog.regprocedure + ) + union all + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_type'::pg_catalog.regclass + join pg_catalog.pg_type type_row on type_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = type_row.typnamespace + where namespace.nspname <> 'pg_catalog' + union all + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_operator'::pg_catalog.regclass + join pg_catalog.pg_operator operator_row on operator_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = operator_row.oprnamespace + where namespace.nspname <> 'pg_catalog' + union all + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_collation'::pg_catalog.regclass + join pg_catalog.pg_collation collation_row on collation_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = collation_row.collnamespace + where namespace.nspname <> 'pg_catalog' + ) unsafe_dependency + ), + 'owned_large_objects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_largeobject_metadata metadata + where metadata.lomowner in ((select oid from runtime_role), (select oid from owner_role)) + ), + 'large_object_acl_privileges', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_largeobject_metadata metadata + where exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(metadata.lomacl, pg_catalog.acldefault('L', metadata.lomowner)) + ) acl + where acl.grantee in (0, (select oid from runtime_role), (select oid from owner_role)) + and acl.privilege_type in ('SELECT', 'UPDATE') + ) + ), + 'large_object_mutation_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + cross join (values ({_sql_literal(RUNTIME_DATABASE_ROLE)}), ({_sql_literal(STAGE_OWNER_DATABASE_ROLE)})) scoped_role(name) + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and pg_catalog.has_function_privilege(scoped_role.name, function_row.oid, 'EXECUTE') + ), + 'public_large_object_mutation_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + cross join lateral pg_catalog.aclexplode( + coalesce(function_row.proacl, pg_catalog.acldefault('f', function_row.proowner)) + ) acl + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and acl.grantee = 0 + and acl.privilege_type = 'EXECUTE' + ), + 'parameter_privileges', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_parameter_acl parameter_acl + cross join (values ({_sql_literal(RUNTIME_DATABASE_ROLE)}), ({_sql_literal(STAGE_OWNER_DATABASE_ROLE)})) scoped_role(name) + where pg_catalog.has_parameter_privilege(scoped_role.name, parameter_acl.parname, 'SET') + or pg_catalog.has_parameter_privilege(scoped_role.name, parameter_acl.parname, 'ALTER SYSTEM') + ), + 'proposal_constraint_shape', coalesce(( + select pg_catalog.count(*) = 6 + and pg_catalog.count(*) filter ( + where (constraint_row.conname, constraint_row.contype) in ( + ('kb_proposals_pkey', 'p'), + ('kb_proposals_proposal_type_check', 'c'), + ('kb_proposals_status_check', 'c'), + ('kb_proposals_applied_by_agent_id_fkey', 'f'), + ('kb_proposals_proposed_by_agent_id_fkey', 'f'), + ('kb_proposals_reviewed_by_agent_id_fkey', 'f') + ) + ) = 6 + from pg_catalog.pg_constraint constraint_row + join proposal_table on proposal_table.oid = constraint_row.conrelid + ), false), + 'unsafe_proposal_constraint_dependencies', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_constraint constraint_row + join proposal_table on proposal_table.oid = constraint_row.conrelid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_constraint'::pg_catalog.regclass + and dependency.objid = constraint_row.oid + and dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + join pg_catalog.pg_proc function_row on function_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname <> 'pg_catalog' + ), + 'unsafe_proposal_index_expressions', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_index index_row + join proposal_table on proposal_table.oid = index_row.indrelid + where index_row.indexprs is not null + or ( + index_row.indpred is not null + and pg_catalog.pg_get_expr(index_row.indpred, index_row.indrelid) + <> '(proposed_by_handle IS NOT NULL)' + ) + ), + 'other_scoped_backends', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_stat_activity activity + where activity.pid <> pg_catalog.pg_backend_pid() + and activity.usename in ({_sql_literal(RUNTIME_DATABASE_ROLE)}, {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}) + ), + 'scoped_prepared_xacts', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_prepared_xacts prepared + where prepared.owner in ({_sql_literal(RUNTIME_DATABASE_ROLE)}, {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}) + ), + 'table_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege(current_user, relation.oid, privilege.name) + ), + 'column_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege(current_user, relation.oid, attribute.attnum, privilege.name) + ), + 'unexpected_table_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + left join allowed_select + on allowed_select.nspname = namespace.nspname + and allowed_select.relname = relation.relname + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege(current_user, relation.oid, 'SELECT') + and allowed_select.relname is null + ), + 'unexpected_column_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + left join allowed_select + on allowed_select.nspname = namespace.nspname + and allowed_select.relname = relation.relname + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege(current_user, relation.oid, attribute.attnum, 'SELECT') + and allowed_select.relname is null + ), + 'sequence_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where relation.relkind = 'S' + and pg_catalog.has_sequence_privilege(current_user, relation.oid, privilege.name) + ), + 'unexpected_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join stage_function + where function_row.oid is distinct from stage_function.oid + and pg_catalog.has_function_privilege(current_user, function_row.oid, 'EXECUTE') + ), + 'unexpected_security_definer_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join stage_function + where function_row.prosecdef + and function_row.oid is distinct from stage_function.oid + and pg_catalog.has_function_privilege(current_user, function_row.oid, 'EXECUTE') + ), + 'owner_grant_contract', coalesce( + not pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + pg_catalog.to_regclass('public.agents'), + 'SELECT' + ) + and pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + pg_catalog.to_regclass('kb_stage.kb_proposals'), + 'SELECT' + ) + and not pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + pg_catalog.to_regclass('kb_stage.kb_proposals'), + 'INSERT' + ), + false + ), + 'owner_missing_agent_select_columns', ( + select pg_catalog.count(*)::int + from (values ('id'), ('handle')) required_column(attname) + left join pg_catalog.pg_attribute attribute + on attribute.attrelid = pg_catalog.to_regclass('public.agents') + and attribute.attname = required_column.attname + and attribute.attnum > 0 + and not attribute.attisdropped + where attribute.attnum is null + or not coalesce( + pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + attribute.attrelid, + attribute.attnum, + 'SELECT' + ), + false + ) + ), + 'owner_missing_proposal_insert_columns', ( + select pg_catalog.count(*)::int + from owner_insert_column + left join pg_catalog.pg_attribute attribute + on attribute.attrelid = pg_catalog.to_regclass('kb_stage.kb_proposals') + and attribute.attname = owner_insert_column.attname + and attribute.attnum > 0 + and not attribute.attisdropped + where attribute.attnum is null + or not coalesce( + pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + attribute.attrelid, + attribute.attnum, + 'INSERT' + ), + false + ) + ), + 'owner_table_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + privilege.name + ) + ), + 'owner_unexpected_column_dml_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + cross join (values ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + attribute.attnum, + privilege.name + ) + and not ( + privilege.name = 'INSERT' + and namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' + and attribute.attname in (select attname from owner_insert_column) + ) + ), + 'owner_unexpected_table_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and pg_catalog.has_table_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + 'SELECT' + ) + and not (namespace.nspname = 'kb_stage' and relation.relname = 'kb_proposals') + ), + 'owner_unexpected_column_selects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid + where relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + and pg_catalog.has_column_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + attribute.attnum, + 'SELECT' + ) + and not ( + (namespace.nspname = 'kb_stage' and relation.relname = 'kb_proposals') + or ( + namespace.nspname = 'public' + and relation.relname = 'agents' + and attribute.attname in ('id', 'handle') + ) + ) + ), + 'owner_sequence_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) + where relation.relkind = 'S' + and pg_catalog.has_sequence_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + relation.oid, + privilege.name + ) + ), + 'owner_unexpected_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + join app_schemas on app_schemas.oid = namespace.oid + cross join stage_function + where function_row.oid is distinct from stage_function.oid + and pg_catalog.has_function_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + function_row.oid, + 'EXECUTE' + ) + ) +)::text; +""".strip() + + +def _allowed_read_sql() -> str: + return """ +select pg_catalog.jsonb_build_object( + 'claims_rows_sampled', (select pg_catalog.count(*) from (select 1 from public.claims limit 1) sampled), + 'proposal_rows_sampled', ( + select pg_catalog.count(*) from (select 1 from kb_stage.kb_proposals limit 1) sampled + ) +)::text; +""".strip() + + +def _canary_count_sql(source_ref: str) -> str: + return f"select pg_catalog.count(*)::text from kb_stage.kb_proposals where source_ref = {_sql_literal(source_ref)};" + + +def _stage_sql(run_id: str, source_ref: str) -> str: + return f""" +begin; +with staged as ( + select kb_stage.stage_leoclean_proposal( + 'revise_claim', + 'runtime-permission-verification', + {_sql_literal(source_ref)}, + 'least-privilege transaction rollback verification', + pg_catalog.jsonb_build_object('run_id', {_sql_literal(run_id)}, 'verification', true) + ) as proposal +) +select pg_catalog.jsonb_build_object( + 'agent_id_matches', coalesce( + proposal ->> 'proposed_by_agent_id' = ( + select id::text from public.agents where handle = 'leo' + ), + false + ), + 'proposal', proposal +)::text +from staged; +set constraints all immediate; +rollback; +""".strip() + + +def _transactional(sql: str) -> str: + return f"begin;\n{sql.rstrip(';')};\nrollback;" + + +def _negative_checks(run_id: str, source_ref: str) -> tuple[NegativeCheck, ...]: + nil_uuid = "00000000-0000-0000-0000-000000000000" + return ( + NegativeCheck( + "proposal_insert", + CANONICAL_DATABASE, + _transactional("insert into kb_stage.kb_proposals select * from kb_stage.kb_proposals where false"), + "42501", + ), + NegativeCheck( + "proposal_update", + CANONICAL_DATABASE, + _transactional("update kb_stage.kb_proposals set status = status where false"), + "42501", + ), + NegativeCheck( + "proposal_delete", + CANONICAL_DATABASE, + _transactional("delete from kb_stage.kb_proposals where false"), + "42501", + ), + NegativeCheck( + "canonical_insert", + CANONICAL_DATABASE, + _transactional("insert into public.claims select * from public.claims where false"), + "42501", + ), + NegativeCheck( + "canonical_update", + CANONICAL_DATABASE, + _transactional("update public.claims set id = id where false"), + "42501", + ), + NegativeCheck( + "canonical_delete", + CANONICAL_DATABASE, + _transactional("delete from public.claims where false"), + "42501", + ), + NegativeCheck( + "large_object_creat", + CANONICAL_DATABASE, + _transactional("select pg_catalog.lo_creat(0)"), + "42501", + ), + NegativeCheck( + "large_object_create", + CANONICAL_DATABASE, + _transactional("select pg_catalog.lo_create(0)"), + "42501", + ), + NegativeCheck( + "large_object_from_bytea", + CANONICAL_DATABASE, + _transactional("select pg_catalog.lo_from_bytea(0, ''::bytea)"), + "42501", + ), + NegativeCheck( + "forged_proposer_overload", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.stage_leoclean_proposal(" + f"'revise_claim', 'forged-principal', 'verification', {_sql_literal(source_ref)}, " + f"'must be unavailable', pg_catalog.jsonb_build_object('run_id', {_sql_literal(run_id)}))" + ), + "42883", + ), + NegativeCheck( + "reviewer_approve_function", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.approve_strict_proposal(" + f"'{nil_uuid}'::uuid, 'revise_claim', '{{}}'::jsonb, 'leo', 'must be denied')" + ), + "42501", + ), + NegativeCheck( + "apply_assert_function", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.assert_approved_proposal(" + f"'{nil_uuid}'::uuid, 'revise_claim', '{{}}'::jsonb, 'hash', " + f"'{nil_uuid}'::uuid, '2000-01-01T00:00:00Z'::timestamptz, 'leo')" + ), + "42501", + ), + NegativeCheck( + "apply_finish_function", + CANONICAL_DATABASE, + _transactional( + "select kb_stage.finish_approved_proposal(" + f"'{nil_uuid}'::uuid, 'revise_claim', '{{}}'::jsonb, 'hash', " + f"'{nil_uuid}'::uuid, '2000-01-01T00:00:00Z'::timestamptz, 'leo', 'leo')" + ), + "42501", + ), + NegativeCheck( + "set_role_stage_owner", + CANONICAL_DATABASE, + _transactional("set role leoclean_kb_stage_owner"), + "42501", + ), + NegativeCheck( + "set_role_reviewer", + CANONICAL_DATABASE, + _transactional("set role kb_review"), + "42501", + ), + NegativeCheck( + "set_role_apply", + CANONICAL_DATABASE, + _transactional("set role kb_apply"), + "42501", + ), + NegativeCheck( + "set_role_administrator", + CANONICAL_DATABASE, + _transactional("set role postgres"), + "42501", + ), + NegativeCheck( + "legacy_database_connect", + LEGACY_DATABASE, + "select 1 /* legacy_database_connect */;", + "42501", + expected_connection_denial=True, + ), + NegativeCheck( + "template_database_connect", + TEMPLATE_DATABASE, + "select 1 /* template_database_connect */;", + "42501", + expected_connection_denial=True, + ), + ) + + +def _expect_sqlstate( + check: NegativeCheck, + *, + env: Mapping[str, str], + runner: Runner, +) -> dict[str, str]: + result = _call( + _psql_command(check.database, check.sql), + env=env, + runner=runner, + check=check.name, + ) + if result.returncode == 0: + raise VerificationError( + "negative_permission_unexpectedly_succeeded", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate="success", + ) + + if check.expected_connection_denial: + normalized = result.stderr.casefold() + expected_database = f'permission denied for database "{check.database}"'.casefold() + if expected_database not in normalized or "does not have connect privilege" not in normalized: + raise VerificationError( + "negative_connection_denial_mismatch", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate="startup_error_unclassified", + ) + return { + "check": check.name, + "expected_sqlstate": check.expected_sqlstate, + "observed_error_class": "connect_privilege_denied", + "observed_sqlstate": "not_exposed_by_libpq_startup", + "result": "denied", + } + + observed = SQLSTATE_RE.findall(result.stderr) + unique_observed = sorted(set(observed)) + if not observed: + raise VerificationError( + "negative_permission_sqlstate_missing", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate="missing", + ) + if unique_observed != [check.expected_sqlstate]: + raise VerificationError( + "negative_permission_sqlstate_mismatch", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate=",".join(unique_observed), + ) + return { + "check": check.name, + "expected_sqlstate": check.expected_sqlstate, + "observed_sqlstate": check.expected_sqlstate, + "result": "denied", + } + + +def _assert_identity(identity: dict[str, Any]) -> dict[str, Any]: + expected = { + "database": CANONICAL_DATABASE, + "current_user": RUNTIME_DATABASE_ROLE, + "session_user": RUNTIME_DATABASE_ROLE, + "server_addr": PRIVATE_CLOUDSQL_HOST, + "server_port": PRIVATE_CLOUDSQL_PORT, + "ssl": True, + "system_identifier": EXPECTED_SYSTEM_IDENTIFIER, + } + if any(identity.get(key) != value for key, value in expected.items()): + raise VerificationError("private_ssl_identity_mismatch", "database_identity") + ssl_version = identity.get("ssl_version") + if not isinstance(ssl_version, str) or not ssl_version: + raise VerificationError("ssl_version_missing", "database_identity") + return {**expected, "ssl_version": ssl_version} + + +def _assert_role_posture(posture: dict[str, Any]) -> dict[str, Any]: + expected_booleans = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": True, + "can_replicate": False, + "inherits_privileges": False, + "is_superuser": False, + } + if posture.get("role") != RUNTIME_DATABASE_ROLE: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + for key, expected in expected_booleans.items(): + if posture.get(key) is not expected: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + + connection_limit = posture.get("connection_limit") + membership_edges = posture.get("direct_membership_edges") + if isinstance(connection_limit, bool) or not isinstance(connection_limit, int) or connection_limit != 8: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + if isinstance(membership_edges, bool) or not isinstance(membership_edges, int) or membership_edges != 0: + raise VerificationError("runtime_role_posture_mismatch", "runtime_role_posture") + + return { + "role": RUNTIME_DATABASE_ROLE, + **expected_booleans, + "connection_limit": connection_limit, + "direct_membership_edges": membership_edges, + } + + +def _assert_stage_owner_role_posture(posture: dict[str, Any]) -> dict[str, Any]: + expected_booleans = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": False, + "can_replicate": False, + "inherits_privileges": False, + "is_superuser": False, + } + if posture.get("role") != STAGE_OWNER_DATABASE_ROLE: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + for key, expected in expected_booleans.items(): + if posture.get(key) is not expected: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + + connection_limit = posture.get("connection_limit") + membership_edges = posture.get("direct_membership_edges") + if isinstance(connection_limit, bool) or not isinstance(connection_limit, int) or connection_limit != -1: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + if isinstance(membership_edges, bool) or not isinstance(membership_edges, int) or membership_edges != 0: + raise VerificationError("stage_owner_role_posture_mismatch", "stage_owner_role_posture") + + return { + "role": STAGE_OWNER_DATABASE_ROLE, + **expected_booleans, + "connection_limit": connection_limit, + "direct_membership_edges": membership_edges, + } + + +def _assert_function_privilege_posture(posture: dict[str, Any]) -> dict[str, dict[str, Any]]: + sanitized: dict[str, dict[str, Any]] = {} + for name, signature, expected_exists, expected_execute in FUNCTION_PRIVILEGE_EXPECTATIONS: + observed = posture.get(name) + if not isinstance(observed, dict): + raise VerificationError("function_privilege_posture_mismatch", "function_privilege_posture") + exists = observed.get("exists") + execute = observed.get("execute") + if exists is not expected_exists or execute is not expected_execute: + raise VerificationError("function_privilege_posture_mismatch", "function_privilege_posture") + sanitized[name] = { + "execute": expected_execute, + "exists": expected_exists, + "signature": signature, + } + return sanitized + + +def _assert_stage_function_definition(posture: dict[str, Any]) -> dict[str, Any]: + if posture.get("exists") is not True: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + for field, expected in STAGE_FUNCTION_METADATA_EXPECTATIONS: + observed = posture.get(field) + if type(observed) is not type(expected) or observed != expected: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + if posture.get("argument_modes") is not None: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + if posture.get("configuration") != ["search_path=pg_catalog, pg_temp"]: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + for field in ("acl_exact", "runtime_execute"): + if posture.get(field) is not True: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + + source = posture.get("source") + if not isinstance(source, str) or len(source.encode("utf-8")) > MAX_STAGE_FUNCTION_SOURCE_BYTES: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + source_sha256 = hashlib.sha256(normalize_stage_function_source(source).encode("utf-8")).hexdigest() + if source_sha256 != EXPECTED_STAGE_FUNCTION_SOURCE_SHA256: + raise VerificationError("stage_function_definition_mismatch", "stage_function_definition") + + return { + "acl_exact": True, + "argument_modes": None, + "configuration": ["search_path=pg_catalog, pg_temp"], + "exists": True, + **{field: expected for field, expected in STAGE_FUNCTION_METADATA_EXPECTATIONS}, + "runtime_execute": True, + "signature": STAGE_FUNCTION_SIGNATURE, + "source_sha256": source_sha256, + } + + +def _assert_catalog_privilege_posture(posture: dict[str, Any]) -> dict[str, int | bool]: + sanitized: dict[str, int | bool] = {} + for field in CATALOG_ZERO_COUNT_FIELDS: + value = posture.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value != 0: + raise VerificationError("catalog_privilege_posture_mismatch", "catalog_privilege_posture") + sanitized[field] = 0 + for field in CATALOG_TRUE_FIELDS: + if posture.get(field) is not True: + raise VerificationError("catalog_privilege_posture_mismatch", "catalog_privilege_posture") + sanitized[field] = True + return sanitized + + +def _assert_allowed_reads(readback: dict[str, Any]) -> dict[str, int]: + sanitized: dict[str, int] = {} + for key in ("claims_rows_sampled", "proposal_rows_sampled"): + value = readback.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value not in (0, 1): + raise VerificationError("allowed_read_invalid", "canonical_allowed_reads") + sanitized[key] = value + return sanitized + + +def _assert_stage_result(staged: dict[str, Any], source_ref: str) -> dict[str, Any]: + proposal = staged.get("proposal") + if not isinstance(proposal, dict) or staged.get("agent_id_matches") is not True: + raise VerificationError("staged_proposal_agent_mismatch", "rolled_back_proposal_stage") + expected = { + "proposed_by_handle": "leo", + "source_ref": source_ref, + "status": "pending_review", + } + if any(proposal.get(key) != value for key, value in expected.items()): + raise VerificationError("staged_proposal_contract_mismatch", "rolled_back_proposal_stage") + proposed_by_agent_id = proposal.get("proposed_by_agent_id") + if not isinstance(proposed_by_agent_id, str) or not proposed_by_agent_id: + raise VerificationError("staged_proposal_agent_missing", "rolled_back_proposal_stage") + return {**expected, "agent_id_matches": True, "proposed_by_agent_id": proposed_by_agent_id} + + +def verify_runtime_permissions( + run_id: str, + *, + runner: Runner = subprocess_runner, + base_env: Mapping[str, str] | None = None, + effective_user: str | None = None, + dependency_validator: DependencyValidator = validate_runtime_dependencies, +) -> dict[str, Any]: + try: + run_id = validate_run_id(run_id) + except ValueError: + raise VerificationError("run_id_invalid", "arguments") from None + + if effective_user is None: + try: + effective_user = pwd.getpwuid(os.geteuid()).pw_name + except KeyError: + raise VerificationError("unix_user_unknown", "runtime_user") from None + if effective_user != RUNTIME_UNIX_USER: + raise VerificationError("unexpected_unix_user", "runtime_user") + dependency_validator() + + source = os.environ if base_env is None else base_env + source_ref = f"leo-runtime-permission:{run_id}" + generated_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + with nullcontext(selected_cloudsdk_config_path()) as config_dir: + gcloud_env = _gcloud_environment(source, config_dir) + password = _read_scoped_password(env=gcloud_env, runner=runner) + psql_env = _psql_environment(source, password) + + identity = _assert_identity( + _parse_json_object( + _run_psql_success( + check="database_identity", + database=CANONICAL_DATABASE, + sql=_identity_sql(), + env=psql_env, + runner=runner, + ), + "database_identity", + ) + ) + role_posture = _assert_role_posture( + _parse_json_object( + _run_psql_success( + check="runtime_role_posture", + database=CANONICAL_DATABASE, + sql=_role_posture_sql(), + env=psql_env, + runner=runner, + ), + "runtime_role_posture", + ) + ) + stage_owner_role_posture = _assert_stage_owner_role_posture( + _parse_json_object( + _run_psql_success( + check="stage_owner_role_posture", + database=CANONICAL_DATABASE, + sql=_stage_owner_role_posture_sql(), + env=psql_env, + runner=runner, + ), + "stage_owner_role_posture", + ) + ) + function_privileges = _assert_function_privilege_posture( + _parse_json_object( + _run_psql_success( + check="function_privilege_posture", + database=CANONICAL_DATABASE, + sql=_function_privilege_posture_sql(), + env=psql_env, + runner=runner, + ), + "function_privilege_posture", + ) + ) + stage_function_definition = _assert_stage_function_definition( + _parse_json_object( + _run_psql_success( + check="stage_function_definition", + database=CANONICAL_DATABASE, + sql=_stage_function_definition_sql(), + env=psql_env, + runner=runner, + ), + "stage_function_definition", + ) + ) + catalog_privileges = _assert_catalog_privilege_posture( + _parse_json_object( + _run_psql_success( + check="catalog_privilege_posture", + database=CANONICAL_DATABASE, + sql=_catalog_privilege_posture_sql(), + env=psql_env, + runner=runner, + ), + "catalog_privilege_posture", + ) + ) + allowed_reads = _assert_allowed_reads( + _parse_json_object( + _run_psql_success( + check="canonical_allowed_reads", + database=CANONICAL_DATABASE, + sql=_allowed_read_sql(), + env=psql_env, + runner=runner, + ), + "canonical_allowed_reads", + ) + ) + rows_before = _parse_zero_count( + _run_psql_success( + check="canary_rows_before", + database=CANONICAL_DATABASE, + sql=_canary_count_sql(source_ref), + env=psql_env, + runner=runner, + ), + "canary_rows_before", + ) + staged = _assert_stage_result( + _parse_json_object( + _run_psql_success( + check="rolled_back_proposal_stage", + database=CANONICAL_DATABASE, + sql=_stage_sql(run_id, source_ref), + env=psql_env, + runner=runner, + ), + "rolled_back_proposal_stage", + ), + source_ref, + ) + rows_after = _parse_zero_count( + _run_psql_success( + check="canary_rows_after", + database=CANONICAL_DATABASE, + sql=_canary_count_sql(source_ref), + env=psql_env, + runner=runner, + ), + "canary_rows_after", + ) + + negative_permissions = [ + _expect_sqlstate(check, env=psql_env, runner=runner) for check in _negative_checks(run_id, source_ref) + ] + administrator_secret = _assert_administrator_secret_denied(env=gcloud_env, runner=runner) + + return { + "artifact": ARTIFACT_NAME, + "checks": { + "allowed_reads": allowed_reads, + "catalog_privileges": catalog_privileges, + "canary_rows_after": rows_after, + "canary_rows_before": rows_before, + "function_privileges": function_privileges, + "negative_permissions": negative_permissions, + "role_posture": role_posture, + "stage_function_definition": stage_function_definition, + "stage_owner_role_posture": stage_owner_role_posture, + "rolled_back_proposal": { + **staged, + "transaction": "rolled_back", + }, + }, + "current_tier": REQUIRED_TIER, + "database_identity": identity, + "execution": { + "canonical_writes_committed": False, + "service_independent": True, + "unix_user": RUNTIME_UNIX_USER, + }, + "generated_at_utc": generated_at, + "mode": "live_private_gcp_staging", + "required_tier": REQUIRED_TIER, + "run_id": run_id, + "schema_version": 1, + "secret_access": { + "administrator": { + **administrator_secret, + "secret": ADMINISTRATOR_PASSWORD_SECRET, + }, + "scoped_runtime": { + "result": "readable", + "secret": SCOPED_PASSWORD_SECRET, + "value_retained": False, + }, + }, + "status": "pass", + "safety": { + "canonical_write_committed": False, + "production_promotion_attempted": False, + "proposal_write_committed": False, + "secret_value_retained": False, + "telegram_send_attempted": False, + }, + "target": { + "database": CANONICAL_DATABASE, + "host": PRIVATE_CLOUDSQL_HOST, + "port": PRIVATE_CLOUDSQL_PORT, + "project": PROJECT_ID, + "role": RUNTIME_DATABASE_ROLE, + "sslmode": "verify-ca", + }, + } + + +def failure_receipt(run_id: str, error: VerificationError) -> dict[str, Any]: + return { + "artifact": ARTIFACT_NAME, + "current_tier": LOWER_TIER, + "error": error.as_dict(), + "generated_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"), + "required_tier": REQUIRED_TIER, + "run_id": run_id, + "schema_version": 1, + "status": "fail", + } + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n" + + +def write_receipt(path: Path, payload: Mapping[str, Any]) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + os.fchmod(descriptor, stat.S_IRUSR | stat.S_IWUSR) + with os.fdopen(descriptor, "w", encoding="utf-8", closefd=False) as handle: + handle.write(canonical_json(payload)) + handle.flush() + os.fsync(handle.fileno()) + finally: + os.close(descriptor) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-id", required=True, type=_run_id_arg) + parser.add_argument("--output", type=Path, help="optional sanitized JSON receipt (written mode 0600)") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + receipt = verify_runtime_permissions(args.run_id) + returncode = 0 + except VerificationError as exc: + receipt = failure_receipt(args.run_id, exc) + returncode = 1 + except Exception: + receipt = failure_receipt(args.run_id, VerificationError("internal_error", "verifier")) + returncode = 1 + + if args.output is not None: + try: + write_receipt(args.output, receipt) + except OSError: + receipt = failure_receipt(args.run_id, VerificationError("output_write_failed", "output")) + returncode = 1 + sys.stdout.write(canonical_json(receipt)) + return returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/verify_gcp_leoclean_service_environment.py b/ops/verify_gcp_leoclean_service_environment.py new file mode 100755 index 0000000..d278d7f --- /dev/null +++ b/ops/verify_gcp_leoclean_service_environment.py @@ -0,0 +1,271 @@ +#!/usr/bin/python3 -I +"""Verify the effective Leo service environment without exposing its values.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +PROJECT_ID = "teleo-501523" +PRIVATE_CLOUDSQL_HOST = "10.61.0.3" +PRIVATE_CLOUDSQL_PORT = "5432" +CANONICAL_DATABASE = "teleo_canonical" +RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime" +SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" +ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password" +RUNTIME_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +RUNTIME_CLOUDSDK_CONFIG = "/usr/local/libexec/livingip/leoclean-kb/gcloud-config" +RUNTIME_SSL_MODE = "verify-ca" +RUNTIME_SSL_ROOT_CERT = "/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem" + +FORBIDDEN_EXACT_FIELDS = frozenset( + { + "ALL_PROXY", + "BASH_ENV", + "BASHOPTS", + "CDPATH", + "CURL_CA_BUNDLE", + "CREDENTIALS_DIRECTORY", + "ENV", + "GLOBIGNORE", + "GOOGLE_APPLICATION_CREDENTIALS", + "HTTPS_PROXY", + "HTTP_PROXY", + "IFS", + "NO_PROXY", + "OPENSSL_CONF", + "OPENSSL_ENGINES", + "OPENSSL_MODULES", + "PS4", + "REQUESTS_CA_BUNDLE", + "SHELLOPTS", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SSLKEYLOGFILE", + "GCONV_PATH", + "TELEO_CLOUDSQL_CREDENTIAL_MODE", + "TELEO_KB_CLAIM_BASE_URL", + "TELEO_KB_READ_ATTEMPTS", + "TELEO_MEMORY_INCLUDE_EXCERPTS", + "TELEO_MEMORY_REDACT", + "XDG_CONFIG_HOME", + } +) + + +class VerificationError(RuntimeError): + """A sanitized failure safe to serialize in a public receipt.""" + + def __init__(self, code: str, fields: tuple[str, ...] = ()) -> None: + self.code = code + self.fields = tuple(sorted(set(fields))) + super().__init__(code) + + def as_dict(self) -> dict[str, object]: + payload: dict[str, object] = {"code": self.code} + if self.fields: + payload["fields"] = list(self.fields) + return payload + + +class _SanitizedArgumentParser(argparse.ArgumentParser): + def error(self, _message: str) -> None: + raise VerificationError("invalid_arguments") + + +def validate_pid(value: str | int) -> int: + """Return a positive process id without retaining the input on failure.""" + + if isinstance(value, bool) or not isinstance(value, (str, int)): + raise VerificationError("invalid_pid") + if isinstance(value, str) and (not value.isascii() or not value.isdecimal() or value.startswith("0")): + raise VerificationError("invalid_pid") + try: + pid = int(value) + except (TypeError, ValueError): + raise VerificationError("invalid_pid") from None + if pid <= 0: + raise VerificationError("invalid_pid") + return pid + + +def expected_environment() -> dict[str, str]: + """Build the exact reviewed environment contract.""" + + return { + "CLOUDSDK_CONFIG": RUNTIME_CLOUDSDK_CONFIG, + "PATH": RUNTIME_PATH, + "PGSSLMODE": RUNTIME_SSL_MODE, + "PGSSLROOTCERT": RUNTIME_SSL_ROOT_CERT, + "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", + "TELEO_CANONICAL_CLOUDSQL_DB": CANONICAL_DATABASE, + "TELEO_CLOUDSQL_DB": CANONICAL_DATABASE, + "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK": "0", + "TELEO_CLOUDSQL_HOST": PRIVATE_CLOUDSQL_HOST, + "TELEO_CLOUDSQL_PASSWORD_SECRET": SCOPED_PASSWORD_SECRET, + "TELEO_CLOUDSQL_PORT": PRIVATE_CLOUDSQL_PORT, + "TELEO_CLOUDSQL_USER": RUNTIME_DATABASE_ROLE, + "TELEO_GCP_METADATA_ONLY": "1", + "TELEO_GCP_PROJECT": PROJECT_ID, + "TELEO_KB_MODE": "cloudsql", + } + + +def parse_environment(raw: bytes) -> dict[str, str]: + """Parse a Linux ``/proc/PID/environ`` block and reject ambiguity.""" + + if not isinstance(raw, bytes) or not raw or not raw.endswith(b"\0"): + raise VerificationError("environment_malformed") + + records = raw[:-1].split(b"\0") + if any(not record for record in records): + raise VerificationError("environment_malformed") + + environment: dict[str, str] = {} + try: + for record in records: + encoded_key, separator, encoded_value = record.partition(b"=") + if not separator or not encoded_key: + raise VerificationError("environment_malformed") + key = encoded_key.decode("utf-8", "strict") + value = encoded_value.decode("utf-8", "strict") + if key in environment: + raise VerificationError("environment_malformed") + environment[key] = value + except UnicodeDecodeError: + raise VerificationError("environment_malformed") from None + return environment + + +def _forbidden_field_labels(environment: Mapping[str, str]) -> tuple[str, ...]: + labels: set[str] = set() + for key in environment: + normalized = key.upper() + if normalized.startswith("PG") and normalized not in {"PGSSLMODE", "PGSSLROOTCERT"}: + labels.add("PG*") + if normalized.startswith("CLOUDSDK_") and normalized != "CLOUDSDK_CONFIG": + labels.add("CLOUDSDK_*") + if normalized.startswith("LD_"): + labels.add("LD_*") + if normalized.startswith("BASH_FUNC_"): + labels.add("BASH_FUNC_*") + if normalized.startswith("PYTHON") and normalized not in { + "PYTHONNOUSERSITE", + "PYTHONSAFEPATH", + "PYTHONUNBUFFERED", + }: + labels.add("PYTHON*") + if normalized in FORBIDDEN_EXACT_FIELDS: + labels.add(normalized) + return tuple(sorted(labels)) + + +def validate_environment( + environment: Mapping[str, str], + *, + allow_missing_cloudsdk_config: bool = False, + allow_missing_runtime_security_fields: bool = False, +) -> tuple[str, ...]: + """Validate the effective environment and return only reviewed field names.""" + + if not isinstance(environment, Mapping) or any( + not isinstance(key, str) or not isinstance(value, str) for key, value in environment.items() + ): + raise VerificationError("environment_malformed") + + expected = expected_environment() + if allow_missing_runtime_security_fields: + for field in ( + "CLOUDSDK_CONFIG", + "PGSSLMODE", + "PGSSLROOTCERT", + "PYTHONNOUSERSITE", + "PYTHONSAFEPATH", + ): + if field not in environment: + expected.pop(field) + if allow_missing_cloudsdk_config and "CLOUDSDK_CONFIG" not in environment: + expected.pop("CLOUDSDK_CONFIG") + administrator_secret = ADMINISTRATOR_PASSWORD_SECRET.casefold() + if any(administrator_secret in value.casefold() for value in environment.values()): + raise VerificationError("administrator_secret_reference") + + forbidden_fields = _forbidden_field_labels(environment) + if forbidden_fields: + raise VerificationError("forbidden_environment_fields", forbidden_fields) + + mismatched = tuple( + sorted(key for key, expected_value in expected.items() if environment.get(key) != expected_value) + ) + if mismatched: + raise VerificationError("environment_mismatch", mismatched) + + return tuple(sorted(expected)) + + +def verify_process_environment( + pid: int, + *, + proc_root: Path = Path("/proc"), + allow_missing_cloudsdk_config: bool = False, + allow_missing_runtime_security_fields: bool = False, +) -> dict[str, object]: + """Read and verify one process environment, returning a sanitized receipt.""" + + validated_pid = validate_pid(pid) + raw = (proc_root / str(validated_pid) / "environ").read_bytes() + validated_fields = validate_environment( + parse_environment(raw), + allow_missing_cloudsdk_config=allow_missing_cloudsdk_config, + allow_missing_runtime_security_fields=allow_missing_runtime_security_fields, + ) + return { + "runtime_environment": "scoped", + "status": "pass", + "validated_fields": list(validated_fields), + } + + +def failure_receipt(error: VerificationError) -> dict[str, object]: + return {"error": error.as_dict(), "status": "fail"} + + +def canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n" + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = _SanitizedArgumentParser(description=__doc__, add_help=False) + parser.add_argument("--pid", required=True) + parser.add_argument("--allow-missing-cloudsdk-config", action="store_true") + parser.add_argument("--allow-missing-runtime-security-fields", action="store_true") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + receipt = verify_process_environment( + validate_pid(args.pid), + allow_missing_cloudsdk_config=args.allow_missing_cloudsdk_config, + allow_missing_runtime_security_fields=args.allow_missing_runtime_security_fields, + ) + returncode = 0 + except VerificationError as exc: + receipt = failure_receipt(exc) + returncode = 1 + except Exception: + receipt = failure_receipt(VerificationError("internal_error")) + returncode = 1 + + sys.stdout.write(canonical_json(receipt)) + return returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_gcp_generated_db_direct_claim_suite.py b/scripts/run_gcp_generated_db_direct_claim_suite.py index dc5f465..1265b83 100644 --- a/scripts/run_gcp_generated_db_direct_claim_suite.py +++ b/scripts/run_gcp_generated_db_direct_claim_suite.py @@ -50,8 +50,10 @@ DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service" DEFAULT_HOST = "10.61.0.3" DEFAULT_PROJECT = "teleo-501523" DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" +DEFAULT_SSL_ROOT_CERT = Path("/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem") +DEFAULT_SYSTEM_IDENTIFIER = "7659718422914359312" DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql" -REVIEWED_CLOUDSQL_TOOL_SHA256 = "7d2074a0fc5f0d48fbf8f7905a72ead8f2696c86a041fa43e078fff5500baa51" +REVIEWED_CLOUDSQL_TOOL_SHA256 = "f71d9dd5cfe5dd10ba0b4439a7b9631a0cb824ce6e05b1faa69dc25690f257f5" COUNT_READBACK_RE = re.compile( r"DB readback:\s*claims:\s*`?(?P\d+)`?;\s*" r"sources:\s*`?(?P\d+)`?;\s*" @@ -195,10 +197,13 @@ def run_target_psql( **os.environ, "PGPASSWORD": password, "PGOPTIONS": "-c default_transaction_read_only=on", + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": str(args.ssl_root_cert), } command = [ "psql", - f"host={args.host} port=5432 dbname={args.target_db} user=postgres sslmode=require connect_timeout=8", + f"host={args.host} port=5432 dbname={args.target_db} user=postgres " + f"sslmode=verify-ca sslrootcert={args.ssl_root_cert} connect_timeout=8", "-X", "-Atq", "-v", @@ -235,7 +240,11 @@ rollback; return json.loads(rows[-1]) -def validate_database_identity(identity: dict[str, Any], target_db: str) -> None: +def validate_database_identity( + identity: dict[str, Any], + target_db: str, + expected_system_identifier: str = DEFAULT_SYSTEM_IDENTIFIER, +) -> None: if identity.get("current_database") != target_db: raise bound.CheckpointError("Cloud SQL identity does not match the generated target database") if identity.get("ssl") is not True: @@ -255,8 +264,8 @@ def validate_database_identity(identity: dict[str, Any], target_db: str) -> None ) if not any(address in network for network in private_networks): raise bound.CheckpointError("Cloud SQL target address is not RFC1918-private") - if not identity.get("system_identifier"): - raise bound.CheckpointError("Cloud SQL target system identifier is missing") + if identity.get("system_identifier") != expected_system_identifier: + raise bound.CheckpointError("Cloud SQL target system identifier is not the reviewed instance") def canonical_status(args: argparse.Namespace) -> dict[str, Any]: @@ -350,6 +359,7 @@ def build_cloudsql_wrapper( host: str, project: str, password_secret: str, + ssl_root_cert: Path, run_nonce: str, system_exec_path: str = bound.SYSTEM_EXEC_PATH, ) -> str: @@ -364,18 +374,21 @@ TARGET_HOST={shlex.quote(host)} SOURCE_COMPUTE={shlex.quote(SOURCE_COMPUTE)} PROJECT={shlex.quote(project)} PASSWORD_SECRET={shlex.quote(password_secret)} +SSL_ROOT_CERT={shlex.quote(str(ssl_root_cert))} RUN_NONCE={shlex.quote(run_nonce)} PYTHON={shlex.quote(python)} export PATH={shlex.quote(system_exec_path)} export CLOUDSDK_CONFIG=/home/teleo/.config/gcloud export PGOPTIONS="-c default_transaction_read_only=on" +export PGSSLMODE=verify-ca +export PGSSLROOTCERT="$SSL_ROOT_CERT" INVOCATION_ID="$($PYTHON -c 'import uuid; print(uuid.uuid4())')" if ! PGPASSWORD="$(gcloud secrets versions access latest --secret="$PASSWORD_SECRET" --project="$PROJECT")"; then exit 124 fi export PGPASSWORD trap 'unset PGPASSWORD PGOPTIONS' EXIT -DB_RECEIPT="$(psql "host=$TARGET_HOST port=5432 dbname=$TARGET_DATABASE user=postgres sslmode=require connect_timeout=8" -X -Atq -v ON_ERROR_STOP=1 <<'SQL' +DB_RECEIPT="$(psql "host=$TARGET_HOST port=5432 dbname=$TARGET_DATABASE user=postgres sslmode=verify-ca sslrootcert=$SSL_ROOT_CERT connect_timeout=8" -X -Atq -v ON_ERROR_STOP=1 <<'SQL' begin transaction read only; select jsonb_build_object( 'current_database', current_database(), @@ -419,6 +432,8 @@ $PYTHON "$CLOUDSQL_TOOL" \ --db "$TARGET_DATABASE" \ --canonical-db "$TARGET_DATABASE" \ --project "$PROJECT" \ + --credential-mode clone-readonly \ + --user postgres \ --password-secret "$PASSWORD_SECRET" \ "$@" STATUS=$? @@ -469,6 +484,7 @@ def patch_temp_bridge(args: argparse.Namespace, temp_profile: Path) -> dict[str, project=args.project, password_secret=args.password_secret, run_nonce=run_nonce, + ssl_root_cert=args.ssl_root_cert, ) binding = f""" ## Generated GCP Database Checkpoint @@ -789,6 +805,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--host", default=DEFAULT_HOST) parser.add_argument("--project", default=DEFAULT_PROJECT) parser.add_argument("--password-secret", default=DEFAULT_SECRET) + parser.add_argument("--ssl-root-cert", default=DEFAULT_SSL_ROOT_CERT, type=Path) parser.add_argument("--service", default=DEFAULT_SERVICE) parser.add_argument("--live-profile", default=bound.LIVE_PROFILE, type=Path) parser.add_argument("--chat-id", default=bound.DEFAULT_CHAT_ID) @@ -805,6 +822,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error("--target-db must start with teleo_clone_ and cannot name a live database") if not args.cloudsql_tool.is_file(): parser.error("--cloudsql-tool must exist") + if not args.ssl_root_cert.is_file(): + parser.error("--ssl-root-cert must exist") if not args.manifest_sql.is_file(): parser.error("--manifest-sql must exist") if not args.parity_receipt.is_file(): diff --git a/scripts/run_gcp_generated_db_working_leo_suite.py b/scripts/run_gcp_generated_db_working_leo_suite.py index 81eb247..f2798de 100644 --- a/scripts/run_gcp_generated_db_working_leo_suite.py +++ b/scripts/run_gcp_generated_db_working_leo_suite.py @@ -98,9 +98,7 @@ def load_pinned_receipt(path: Path, expected_sha256: str, artifact: str) -> tupl return payload, actual_sha256 -def validate_target_identity_receipt( - args: argparse.Namespace, path: Path, expected_sha256: str -) -> dict[str, Any]: +def validate_target_identity_receipt(args: argparse.Namespace, path: Path, expected_sha256: str) -> dict[str, Any]: payload, actual_sha256 = load_pinned_receipt(path, expected_sha256, TARGET_IDENTITY_ARTIFACT) required = { "target_database": args.target_db, @@ -130,9 +128,7 @@ def validate_target_identity_receipt( } -def validate_source_baseline_receipt( - args: argparse.Namespace, path: Path, expected_sha256: str -) -> dict[str, Any]: +def validate_source_baseline_receipt(args: argparse.Namespace, path: Path, expected_sha256: str) -> dict[str, Any]: payload, actual_sha256 = load_pinned_receipt(path, expected_sha256, SOURCE_BASELINE_ARTIFACT) if payload.get("project") != args.project or payload.get("instance") != args.instance: raise bound.CheckpointError("source baseline receipt project/instance context does not match") @@ -192,8 +188,7 @@ def composition_check_contract() -> list[str]: if "source_packet_validated" in keys and "isolated_restart_and_memory_survived" in keys: if len(keys) != EXPECTED_COMPOSITION_CHECK_COUNT: raise bound.CheckpointError( - "composition scorer drifted: " - f"expected {EXPECTED_COMPOSITION_CHECK_COUNT} checks, found {len(keys)}" + f"composition scorer drifted: expected {EXPECTED_COMPOSITION_CHECK_COUNT} checks, found {len(keys)}" ) return keys raise bound.CheckpointError("existing composition scorer contract could not be located") @@ -334,8 +329,7 @@ def validate_capability_receipt( "session_user": receipt.get("session_user") == role, "current_user": receipt.get("current_user") == role, "current_database": receipt.get("current_database") == args.target_db, - "default_transaction_read_only": receipt.get("default_transaction_read_only") - == ("off" if writable else "on"), + "default_transaction_read_only": receipt.get("default_transaction_read_only") == ("off" if writable else "on"), "not_superuser": receipt.get("is_superuser") is False, "can_select_proposals": receipt.get("can_select_proposals") is True, "cannot_update_proposals": receipt.get("can_update_proposals") is False, @@ -370,12 +364,7 @@ def validate_capability_receipt( "can_insert_canonical": True, }, } - checks.update( - { - name: receipt.get(name) is expected - for name, expected in expected_capabilities[purpose].items() - } - ) + checks.update({name: receipt.get(name) is expected for name, expected in expected_capabilities[purpose].items()}) failed = sorted(name for name, passed in checks.items() if not passed) if failed: raise bound.CheckpointError(f"{purpose} phase capability receipt failed: " + ", ".join(failed)) @@ -452,7 +441,8 @@ select current_database() = :'expected_database' as exact_database_bound \\gset "psql", ( f"host={self.args.host} port=5432 dbname={self.args.target_db} " - f"user={role} sslmode=require connect_timeout=8" + f"user={role} sslmode=verify-ca " + f"sslrootcert={self.args.ssl_root_cert} connect_timeout=8" ), "-X", "-Atq", @@ -461,7 +451,13 @@ select current_database() = :'expected_database' as exact_database_bound \\gset "-v", f"expected_database={self.args.target_db}", ] - env = {**os.environ, "PGPASSWORD": password, "PGOPTIONS": pgoptions} + env = { + **os.environ, + "PGPASSWORD": password, + "PGOPTIONS": pgoptions, + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": str(self.args.ssl_root_cert), + } try: try: output = gcp.run( @@ -474,6 +470,8 @@ select current_database() = :'expected_database' as exact_database_bound \\gset finally: env.pop("PGPASSWORD", None) env.pop("PGOPTIONS", None) + env.pop("PGSSLMODE", None) + env.pop("PGSSLROOTCERT", None) password = "" lines = output.splitlines() receipt_lines = [line for line in lines if line.startswith(ROLE_RECEIPT_PREFIX)] @@ -572,7 +570,9 @@ def verify_distinct_credential_values(executor: CloudSqlExecutor) -> dict[str, A values.clear() -def _completed(command: list[str], returncode: int, stdout: str = "", stderr: str = "") -> subprocess.CompletedProcess[str]: +def _completed( + command: list[str], returncode: int, stdout: str = "", stderr: str = "" +) -> subprocess.CompletedProcess[str]: return subprocess.CompletedProcess(command, returncode, stdout, stderr) @@ -643,13 +643,10 @@ def _target_identity(args: argparse.Namespace, db_identity: dict[str, Any]) -> d } -def assert_live_target_identity( - args: argparse.Namespace, retained: dict[str, Any], live: dict[str, Any] -) -> None: +def assert_live_target_identity(args: argparse.Namespace, retained: dict[str, Any], live: dict[str, Any]) -> None: checks = { "target_database": live.get("current_database") == retained.get("target_database") == args.target_db, - "system_identifier": str(live.get("system_identifier") or "") - == str(retained.get("system_identifier") or ""), + "system_identifier": str(live.get("system_identifier") or "") == str(retained.get("system_identifier") or ""), "server_address": str(live.get("server_address") or "") == str(retained.get("server_address") or "") == args.host, @@ -692,6 +689,7 @@ def _build_gcp_read_wrapper( host=args.host, project=args.project, password_secret=args.password_secret, + ssl_root_cert=getattr(args, "ssl_root_cert", gcp.DEFAULT_SSL_ROOT_CERT), run_nonce=run_nonce, ) wrapper = wrapper.replace( @@ -699,7 +697,11 @@ def _build_gcp_read_wrapper( f"PASSWORD_SECRET={shlex.quote(args.password_secret)}\nREAD_ROLE={shlex.quote(args.read_role)}\n", 1, ) - wrapper = wrapper.replace("user=postgres sslmode=require", "user=$READ_ROLE sslmode=require", 1) + wrapper = wrapper.replace( + "user=postgres sslmode=verify-ca", + "user=$READ_ROLE sslmode=verify-ca", + 1, + ) wrapper = wrapper.replace( " 'default_transaction_read_only', current_setting('default_transaction_read_only')\n", " 'default_transaction_read_only', current_setting('default_transaction_read_only'),\n" @@ -737,7 +739,7 @@ def _cloudsql_lifecycle_wrapper( tool_log=tool_log, run_nonce=run_nonce, ) - marker = "if ! PGPASSWORD=\"$(gcloud secrets versions access latest" + marker = 'if ! PGPASSWORD="$(gcloud secrets versions access latest' if marker not in base: raise bound.CheckpointError("reviewed Cloud SQL wrapper shape changed; lifecycle injection refused") internal = shlex.join( @@ -755,6 +757,8 @@ def _cloudsql_lifecycle_wrapper( args.instance, "--password-secret", args.password_secret, + "--ssl-root-cert", + str(args.ssl_root_cert), "--read-role", args.read_role, "--stage-password-secret", @@ -866,12 +870,16 @@ class RuntimeAdapter: self._assert_target(container, database) return self.executor.read(sql) - def _approve(self, _guard_args: argparse.Namespace, proposal_id: str, database: str, *, dry_run: bool = False) -> subprocess.CompletedProcess[str]: + def _approve( + self, _guard_args: argparse.Namespace, proposal_id: str, database: str, *, dry_run: bool = False + ) -> subprocess.CompletedProcess[str]: if database != self.args.target_db: raise bound.CheckpointError("review attempted database fallback") return _review_command(self.executor, proposal_id, dry_run=dry_run) - def _apply(self, _guard_args: argparse.Namespace, proposal_id: str, database: str, *, dry_run: bool = False) -> subprocess.CompletedProcess[str]: + def _apply( + self, _guard_args: argparse.Namespace, proposal_id: str, database: str, *, dry_run: bool = False + ) -> subprocess.CompletedProcess[str]: if database != self.args.target_db: raise bound.CheckpointError("apply attempted database fallback") return _apply_command(self.executor, proposal_id, dry_run=dry_run) @@ -919,9 +927,7 @@ class RuntimeAdapter: self._restore() raise - def _assert_disposable_target( - self, container: str, database: str - ) -> tuple[dict[str, Any], dict[str, Any]]: + def _assert_disposable_target(self, container: str, database: str) -> tuple[dict[str, Any], dict[str, Any]]: self._assert_target(container, database) target = self._container_identity(container) source = self._container_identity(bound.PRODUCTION_CONTAINER) @@ -1185,9 +1191,7 @@ def catalog_session_isolation(results: list[dict[str, Any]]) -> dict[str, bool]: dc = [result for result in results if str(result.get("prompt_id") or "").startswith("DC-")] session_ids = [result.get("persisted_session_id") for result in dc] return { - "all_profiles_unique": bool(profile_ids) - and all(profile_ids) - and len(profile_ids) == len(set(profile_ids)), + "all_profiles_unique": bool(profile_ids) and all(profile_ids) and len(profile_ids) == len(set(profile_ids)), "dc_isolated": len(dc) == len(benchmark.M3TAVERSAL_DIRECT_CLAIM_FOLLOWUP_SCENARIOS) and all(result.get("fresh_private_profile") is True and result.get("prior_prompt_ids") == [] for result in dc) and all(session_ids) @@ -1432,9 +1436,7 @@ async def run_live_suite(args: argparse.Namespace, validated: dict[str, Any]) -> report[name] = capture() except Exception as exc: report[name] = None - report["errors"].append( - {"phase": name, "type": type(exc).__name__, "message": safe_message(exc)} - ) + report["errors"].append({"phase": name, "type": type(exc).__name__, "message": safe_message(exc)}) report["database_fingerprint_after"] = ( database_fingerprint_from_manifest(report["database_manifest_after"]) if report.get("database_manifest_after") @@ -1493,12 +1495,10 @@ async def run_live_suite(args: argparse.Namespace, validated: dict[str, Any]) -> "source_baseline_is_independently_retained_and_physically_distinct": bool(source_baseline.get("sha256")) and source_identity.get("system_identifier") and str(source_identity.get("system_identifier")) != str(before_identity.get("system_identifier")), - "proposal_lifecycle_exact_under_gcp_supported_checks": report["lifecycle_gcp_evaluation"]["passed"] - is True, + "proposal_lifecycle_exact_under_gcp_supported_checks": report["lifecycle_gcp_evaluation"]["passed"] is True, "process_memory_survived": (composition_checks.get("isolated_restart_and_memory_survived") is True) and (lifecycle_checks.get("isolated_handler_reopened_with_same_session") is True), - "database_identity_stable": bool(before_identity) - and before_identity == report.get("database_identity_after"), + "database_identity_stable": bool(before_identity) and before_identity == report.get("database_identity_after"), "physical_target_matches_sha_pinned_receipt": bool(target_identity.get("sha256")) and str(before_identity.get("system_identifier")) == target_identity.get("system_identifier"), "service_unchanged": before_service == report.get("service_after"), @@ -1509,12 +1509,8 @@ async def run_live_suite(args: argparse.Namespace, validated: dict[str, Any]) -> ) is True and report["object_level_changes"].get("lifecycle_unrelated_rows_unchanged") is True - and all( - item.get("changed") is True - for item in (report["object_level_changes"].get("objects") or {}).values() - ), - "phase_roles_and_capabilities_verified": {"read", "stage", "review", "apply"} - <= operation_purposes + and all(item.get("changed") is True for item in (report["object_level_changes"].get("objects") or {}).values()), + "phase_roles_and_capabilities_verified": {"read", "stage", "review", "apply"} <= operation_purposes and bool(executor.receipts) and all( ( @@ -1522,18 +1518,12 @@ async def run_live_suite(args: argparse.Namespace, validated: dict[str, Any]) -> and receipt["purpose"] in ALLOWED_WRITE_PURPOSES and receipt["default_transaction_read_only"] == "off" ) - or ( - receipt["writable"] is False - and receipt["default_transaction_read_only"] == "on" - ) + or (receipt["writable"] is False and receipt["default_transaction_read_only"] == "on") for receipt in executor.receipts ), "credential_values_verified_distinct_without_retention": report["credential_value_separation"] == {"checked": True, "distinct": True, "credential_count": 4}, - "no_send_proven_by_component_tool_surfaces": catalog_checks.get( - "send_and_delivery_absent_per_prompt" - ) - is True + "no_send_proven_by_component_tool_surfaces": catalog_checks.get("send_and_delivery_absent_per_prompt") is True and composition_checks.get("no_telegram_send") is True and lifecycle_checks.get("no_telegram_send") is True, "no_systemd_restart": report["systemd_restart_attempted"] is False, @@ -1600,6 +1590,7 @@ def parse_internal_stage_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--project", required=True) parser.add_argument("--instance", required=True) parser.add_argument("--password-secret", required=True) + parser.add_argument("--ssl-root-cert", required=True, type=Path) parser.add_argument("--read-role", required=True) parser.add_argument("--stage-password-secret", required=True) parser.add_argument("--stage-role", required=True) @@ -1609,6 +1600,8 @@ def parse_internal_stage_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--target-identity-receipt", required=True, type=Path) parser.add_argument("--target-identity-sha256", required=True) args = parser.parse_args(argv) + if not args.ssl_root_cert.is_file(): + parser.error("--ssl-root-cert must exist") args.review_role = "kb_review" args.apply_role = "kb_apply" args.allow_stage = True @@ -1680,6 +1673,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--project", default=gcp.DEFAULT_PROJECT) parser.add_argument("--instance") parser.add_argument("--password-secret", default=gcp.DEFAULT_SECRET) + parser.add_argument("--ssl-root-cert", default=gcp.DEFAULT_SSL_ROOT_CERT, type=Path) parser.add_argument("--read-role", default="kb_read") parser.add_argument("--stage-role", default=DEFAULT_STAGE_ROLE) parser.add_argument("--review-role", default=DEFAULT_REVIEW_ROLE) @@ -1713,6 +1707,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: if args.turn_timeout <= 0: parser.error("--turn-timeout must be positive") if args.execute: + if not args.ssl_root_cert.is_file(): + parser.error("--ssl-root-cert must exist for execution") receipt_inputs = ( args.target_identity_receipt, args.target_identity_sha256, @@ -1721,9 +1717,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: args.instance, ) if not all(receipt_inputs): - parser.error( - "execution requires --instance plus SHA-pinned target identity and source baseline receipts" - ) + parser.error("execution requires --instance plus SHA-pinned target identity and source baseline receipts") missing_flags = [name for name in ("stage", "review", "apply") if not getattr(args, f"allow_{name}")] if missing_flags: parser.error("execution requires explicit controller flags: " + ", ".join(missing_flags)) diff --git a/scripts/run_leo_negative_permissions_canary.py b/scripts/run_leo_negative_permissions_canary.py new file mode 100644 index 0000000..76403b3 --- /dev/null +++ b/scripts/run_leo_negative_permissions_canary.py @@ -0,0 +1,807 @@ +#!/usr/bin/env python3 +"""Run a networkless PostgreSQL least-privilege lifecycle for Leo. + +The canary provisions the real guarded proposal/apply prerequisites and the +real Observatory read role in a disposable PostgreSQL container. It then +connects as the read principal, proves required retrieval, attempts forbidden +database and role operations, compares catalog/data fingerprints, and verifies +that the container and temporary work directory were removed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import secrets +import shutil +import subprocess +import tempfile +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +APPLY_PREREQS_SQL = ROOT / "scripts" / "kb_apply_prereqs.sql" +READ_ROLE_SQL = ROOT / "ops" / "observatory_read_role.sql" +DATABASE = "teleo_canonical" +IAM_USER = "sa-observatory-read-adapter@teleo-501523.iam" +PARTICIPANT_HANDLE = "m3taversal" +DEFAULT_IMAGE = "postgres:16-alpine" +CANARY_LABEL = "leo-negative-permissions" + +BOOTSTRAP_SQL = r""" +create role kb_gate_owner nologin inherit; +create role kb_review login inherit; +create role kb_apply login inherit; + +create schema kb_stage; + +create table public.agents ( + id uuid primary key, + handle text not null unique, + kind text not null +); +create table public.strategies ( + id uuid primary key, + agent_id uuid not null, + active boolean not null default true +); +create table public.strategy_nodes (id uuid primary key); +create table public.claim_evidence ( + id uuid primary key, + claim_id uuid not null, + source_id uuid not null, + role text not null +); +create table public.claim_edges (id uuid primary key); +create table public.claims ( + id uuid primary key, + text text not null +); +create table public.sources ( + id uuid primary key, + hash text not null +); +create table public.reasoning_tools (id uuid primary key); +create table public.private_notes ( + id uuid primary key, + note text not null +); + +insert into public.agents (id, handle, kind) +values ('11111111-1111-1111-1111-111111111111', '__PARTICIPANT_HANDLE__', 'human'); + +create table kb_stage.kb_proposals ( + id uuid primary key, + proposal_type text not null, + status text not null, + payload jsonb not null, + reviewed_by_handle text, + reviewed_by_agent_id uuid, + reviewed_at timestamptz, + review_note text, + applied_by_handle text, + applied_by_agent_id uuid, + applied_at timestamptz, + updated_at timestamptz not null default now() +); +create table kb_stage.kb_review_principals ( + db_role name primary key, + reviewed_by_handle text not null, + reviewed_by_agent_id uuid not null references public.agents(id), + active boolean not null default true, + created_at timestamptz not null default now() +); +create table kb_stage.kb_proposal_approvals ( + proposal_id uuid primary key references kb_stage.kb_proposals(id), + proposal_type text not null, + payload jsonb not null, + reviewed_by_handle text not null, + reviewed_by_agent_id uuid not null references public.agents(id), + reviewed_by_db_role name not null, + reviewed_at timestamptz not null, + review_note text not null +); + +insert into public.claims (id, text) +values ( + '22222222-2222-2222-2222-222222222222', + 'Canonical Leo knowledge remains review-gated.' +); +insert into public.sources (id, hash) +values ('33333333-3333-3333-3333-333333333333', 'fixture-source-hash'); +insert into public.private_notes (id, note) +values ('44444444-4444-4444-4444-444444444444', 'not allowlisted'); +insert into kb_stage.kb_proposals ( + id, proposal_type, status, payload +) values ( + '55555555-5555-5555-5555-555555555555', + 'revise_strategy', + 'pending_review', + '{"apply_payload":{"agent_id":"11111111-1111-1111-1111-111111111111"}}' +); +""" + + +class CanaryError(RuntimeError): + """A setup or verification step failed.""" + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def canonical_sha256(value: Any) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def sql_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def sql_identifier(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +def run( + command: list[str], + *, + input_text: str | None = None, + check: bool = True, + timeout: int = 60, +) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + command, + input=input_text, + text=True, + capture_output=True, + check=False, + timeout=timeout, + ) + if check and completed.returncode != 0: + stderr = completed.stderr.strip()[-2000:] + raise CanaryError(f"command failed with exit {completed.returncode}: {stderr}") + return completed + + +def admin_psql(container: str, sql: str, *, exec_env: dict[str, str] | None = None) -> str: + command = ["docker", "exec"] + for name, value in sorted((exec_env or {}).items()): + command.extend(["-e", f"{name}={value}"]) + command.extend( + [ + "-i", + container, + "psql", + "-X", + "--set", + "ON_ERROR_STOP=1", + "-U", + "postgres", + "-d", + DATABASE, + "-At", + ] + ) + return run(command, input_text=sql).stdout + + +def user_psql( + container: str, + password: str, + sql_commands: list[str], +) -> subprocess.CompletedProcess[str]: + command = [ + "docker", + "exec", + "-e", + f"PGPASSWORD={password}", + container, + "psql", + "-X", + "--set", + "ON_ERROR_STOP=1", + "--set", + "VERBOSITY=verbose", + "-h", + "127.0.0.1", + "-U", + IAM_USER, + "-d", + DATABASE, + "-At", + ] + for sql in sql_commands: + command.extend(["-c", sql]) + return run(command, check=False) + + +def parse_last_json(output: str) -> dict[str, Any]: + for line in reversed(output.splitlines()): + line = line.strip() + if not line.startswith("{"): + continue + parsed = json.loads(line) + if isinstance(parsed, dict): + return parsed + raise CanaryError("expected a JSON object in psql output") + + +def sanitized_denial(completed: subprocess.CompletedProcess[str]) -> str: + lines = [line.strip() for line in completed.stderr.splitlines() if line.strip()] + for line in lines: + if "ERROR:" in line or "FATAL:" in line: + return line[:500] + return (lines[-1] if lines else "no database denial was returned")[:500] + + +def allowed_operation( + operation_id: str, + category: str, + completed: subprocess.CompletedProcess[str], +) -> dict[str, Any]: + return { + "id": operation_id, + "category": category, + "expected": "allowed", + "passed": completed.returncode == 0, + "exit_code": completed.returncode, + "readback": [line for line in completed.stdout.splitlines() if line], + "error": sanitized_denial(completed) if completed.returncode != 0 else None, + } + + +def denied_operation( + operation_id: str, + category: str, + completed: subprocess.CompletedProcess[str], + expected_fragments: tuple[str, ...], +) -> dict[str, Any]: + denial = sanitized_denial(completed) + lowered = denial.lower() + matched = next((fragment for fragment in expected_fragments if fragment.lower() in lowered), None) + return { + "id": operation_id, + "category": category, + "expected": "denied", + "passed": completed.returncode != 0 and matched is not None, + "exit_code": completed.returncode, + "matched_denial": matched, + "denial": denial, + } + + +def fingerprint(container: str) -> dict[str, Any]: + principal = sql_literal(IAM_USER) + query = f""" +select jsonb_build_object( + 'database', current_database(), + 'participant_handle', ( + select handle + from public.agents + where id = '11111111-1111-1111-1111-111111111111' + ), + 'principal', ( + select jsonb_build_object( + 'rolname', rolname, + 'rolinherit', rolinherit, + 'rolsuper', rolsuper, + 'rolcreatedb', rolcreatedb, + 'rolcreaterole', rolcreaterole, + 'rolreplication', rolreplication, + 'rolbypassrls', rolbypassrls + ) + from pg_catalog.pg_roles + where rolname = {principal} + ), + 'memberships', coalesce(( + select jsonb_agg( + jsonb_build_object( + 'role', granted.rolname, + 'admin_option', membership.admin_option + ) order by granted.rolname + ) + from pg_catalog.pg_auth_members membership + join pg_catalog.pg_roles granted on granted.oid = membership.roleid + join pg_catalog.pg_roles member on member.oid = membership.member + where member.rolname = {principal} + ), '[]'::jsonb), + 'role_database_settings', coalesce(( + select jsonb_agg(setting order by setting) + from pg_catalog.pg_db_role_setting role_setting + cross join lateral unnest(role_setting.setconfig) setting + where role_setting.setrole = (select oid from pg_catalog.pg_roles where rolname = {principal}) + and role_setting.setdatabase = (select oid from pg_catalog.pg_database where datname = current_database()) + ), '[]'::jsonb), + 'effective_privileges', jsonb_build_object( + 'claims_select', pg_catalog.has_table_privilege({principal}, 'public.claims', 'SELECT'), + 'claims_insert', pg_catalog.has_table_privilege({principal}, 'public.claims', 'INSERT'), + 'proposals_select', pg_catalog.has_table_privilege({principal}, 'kb_stage.kb_proposals', 'SELECT'), + 'proposals_insert', pg_catalog.has_table_privilege({principal}, 'kb_stage.kb_proposals', 'INSERT'), + 'private_notes_select', pg_catalog.has_table_privilege({principal}, 'public.private_notes', 'SELECT'), + 'public_schema_create', pg_catalog.has_schema_privilege({principal}, 'public', 'CREATE'), + 'kb_stage_schema_create', pg_catalog.has_schema_privilege({principal}, 'kb_stage', 'CREATE'), + 'approve_execute', pg_catalog.has_function_privilege( + {principal}, + 'kb_stage.approve_strict_proposal(uuid,text,jsonb,text,text)', + 'EXECUTE' + ), + 'assert_execute', pg_catalog.has_function_privilege( + {principal}, + 'kb_stage.assert_approved_proposal(uuid,text,jsonb,text,uuid,timestamptz,text)', + 'EXECUTE' + ), + 'finish_execute', pg_catalog.has_function_privilege( + {principal}, + 'kb_stage.finish_approved_proposal(uuid,text,jsonb,text,uuid,timestamptz,text,text)', + 'EXECUTE' + ) + ), + 'row_counts', jsonb_build_object( + 'public.claims', (select count(*) from public.claims), + 'public.sources', (select count(*) from public.sources), + 'public.claim_evidence', (select count(*) from public.claim_evidence), + 'public.claim_edges', (select count(*) from public.claim_edges), + 'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals) + ) +)::text; +""" + return parse_last_json(admin_psql(container, query)) + + +def wait_until_ready(container: str) -> None: + ready_streak = 0 + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + ready = run( + ["docker", "exec", container, "pg_isready", "-U", "postgres", "-d", DATABASE], + check=False, + ) + if ready.returncode == 0: + ready_streak += 1 + if ready_streak >= 2: + return + else: + ready_streak = 0 + time.sleep(0.25) + raise CanaryError("disposable PostgreSQL did not become stably ready") + + +def operation_suite(container: str, password: str) -> list[dict[str, Any]]: + read_only_off = "set default_transaction_read_only = off" + fake_payload = "'{\"apply_payload\":{}}'::jsonb" + proposal_id = "'55555555-5555-5555-5555-555555555555'::uuid" + reviewer_id = "'11111111-1111-1111-1111-111111111111'::uuid" + reviewer_handle = sql_literal(PARTICIPANT_HANDLE) + operations = [ + allowed_operation( + "session_read_only_defaults", + "session", + user_psql( + container, + password, + [ + "select jsonb_build_object(" + "'default_transaction_read_only', current_setting('default_transaction_read_only'), " + "'transaction_read_only', current_setting('transaction_read_only'))::text" + ], + ), + ), + allowed_operation( + "canonical_retrieval", + "retrieval", + user_psql( + container, + password, + [ + "select jsonb_build_object('claim_count', count(*), 'sample_text', min(text))::text " + "from public.claims" + ], + ), + ), + allowed_operation( + "proposal_ledger_retrieval", + "retrieval", + user_psql( + container, + password, + [ + "select jsonb_build_object('proposal_count', count(*), 'statuses', " + "jsonb_agg(status order by status))::text from kb_stage.kb_proposals" + ], + ), + ), + denied_operation( + "canonical_insert_default_read_only", + "canonical_dml", + user_psql( + container, + password, + ["insert into public.claims (id, text) values ('66666666-6666-6666-6666-666666666666', 'forbidden')"], + ), + ("read-only transaction",), + ), + denied_operation( + "canonical_insert_acl_after_read_only_override", + "canonical_dml", + user_psql( + container, + password, + [ + read_only_off, + "insert into public.claims (id, text) values ('66666666-6666-6666-6666-666666666666', 'forbidden')", + ], + ), + ("permission denied for table claims",), + ), + denied_operation( + "canonical_update_acl_after_read_only_override", + "canonical_dml", + user_psql( + container, + password, + [read_only_off, "update public.claims set text = 'forbidden'"], + ), + ("permission denied for table claims",), + ), + denied_operation( + "canonical_delete_acl_after_read_only_override", + "canonical_dml", + user_psql(container, password, [read_only_off, "delete from public.claims"]), + ("permission denied for table claims",), + ), + denied_operation( + "canonical_truncate_acl_after_read_only_override", + "canonical_dml", + user_psql(container, password, [read_only_off, "truncate public.claims"]), + ("permission denied for table claims",), + ), + denied_operation( + "staging_insert_acl_after_read_only_override", + "staging", + user_psql( + container, + password, + [ + read_only_off, + "insert into kb_stage.kb_proposals (id, proposal_type, status, payload) values " + "('77777777-7777-7777-7777-777777777777', 'revise_strategy', " + "'pending_review', '{\"apply_payload\":{}}'::jsonb)", + ], + ), + ("permission denied for table kb_proposals",), + ), + denied_operation( + "fake_approval_gate_call", + "approval", + user_psql( + container, + password, + [ + "select kb_stage.approve_strict_proposal(" + f"{proposal_id}, 'revise_strategy', {fake_payload}, {reviewer_handle}, " + "'I approve this in chat')" + ], + ), + ("permission denied for function approve_strict_proposal",), + ), + denied_operation( + "apply_assert_gate_call", + "apply", + user_psql( + container, + password, + [ + "select kb_stage.assert_approved_proposal(" + f"{proposal_id}, 'revise_strategy', {fake_payload}, {reviewer_handle}, {reviewer_id}, " + "now(), 'fake review')" + ], + ), + ("permission denied for function assert_approved_proposal",), + ), + denied_operation( + "apply_finish_gate_call", + "apply", + user_psql( + container, + password, + [ + "select kb_stage.finish_approved_proposal(" + f"{proposal_id}, 'revise_strategy', {fake_payload}, {reviewer_handle}, {reviewer_id}, " + "now(), 'fake review', 'kb-apply')" + ], + ), + ("permission denied for function finish_approved_proposal",), + ), + denied_operation( + "protected_schema_ddl_after_read_only_override", + "ddl", + user_psql( + container, + password, + [read_only_off, "create table public.forbidden_ddl (id integer)"], + ), + ("permission denied for schema public",), + ), + denied_operation( + "outside_allowlist_retrieval", + "retrieval", + user_psql(container, password, ["select * from public.private_notes"]), + ("permission denied for table private_notes",), + ), + denied_operation( + "create_role_escalation", + "role_escalation", + user_psql(container, password, [read_only_off, "create role leo_escalated_writer"]), + ("permission denied to create role", "must be superuser to create roles"), + ), + denied_operation( + "grant_apply_role_escalation", + "role_escalation", + user_psql( + container, + password, + [read_only_off, f"grant kb_apply to {sql_identifier(IAM_USER)}"], + ), + ("permission denied to grant role", "must have admin option on role"), + ), + denied_operation( + "set_apply_role_escalation", + "role_escalation", + user_psql(container, password, ["set role kb_apply"]), + ("permission denied to set role",), + ), + denied_operation( + "alter_role_createdb_escalation", + "role_escalation", + user_psql( + container, + password, + [read_only_off, f"alter role {sql_identifier(IAM_USER)} createdb"], + ), + ("permission denied to alter role", "must be superuser to alter"), + ), + denied_operation( + "claim_table_ownership_escalation", + "role_escalation", + user_psql( + container, + password, + [read_only_off, f"alter table public.claims owner to {sql_identifier(IAM_USER)}"], + ), + ("must be owner of table claims",), + ), + ] + return operations + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True, help="Receipt JSON destination") + parser.add_argument("--image", default=DEFAULT_IMAGE, help="Disposable PostgreSQL image") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if shutil.which("docker") is None: + raise SystemExit("Docker is required for the negative-permissions canary") + + run_id = uuid.uuid4().hex[:12] + container = f"leo-negative-permissions-{run_id}" + temp_dir = Path(tempfile.mkdtemp(prefix=f"{container}-")) + password = secrets.token_urlsafe(24) + started_at = utc_now() + container_started = False + execution_passed = False + receipt: dict[str, Any] = { + "artifact": "leo_negative_permissions_canary", + "schema": "livingip.leo-negative-permissions.v1", + "run_id": run_id, + "started_at_utc": started_at, + "required_tier": "T2_runtime", + "current_tier": "T0_spec", + "scope": { + "database": DATABASE, + "principal": IAM_USER, + "container_network": "none", + "production_contacted": False, + "canonical_production_rows_contacted": False, + }, + "identity_policy": { + "participant_handle": PARTICIPANT_HANDLE, + "reviewer_handle": PARTICIPANT_HANDLE, + "exact_handle_required": True, + }, + "operations": [], + "passed": False, + } + + try: + repo_sha = run(["git", "rev-parse", "HEAD"], timeout=15).stdout.strip() + docker_server = run(["docker", "version", "--format", "{{.Server.Version}}"], timeout=15).stdout.strip() + image_id = run( + ["docker", "image", "inspect", "--format", "{{.Id}}", args.image], + timeout=15, + ).stdout.strip() + receipt["runtime"] = { + "repo_git_sha": repo_sha, + "postgres_image": args.image, + "postgres_image_id": image_id, + "docker_server_version": docker_server, + } + + run( + [ + "docker", + "run", + "--detach", + "--rm", + "--name", + container, + "--network", + "none", + "--label", + f"livingip.canary={CANARY_LABEL}", + "--label", + f"livingip.canary.instance={container}", + "--tmpfs", + "/var/lib/postgresql/data:rw,nosuid,nodev,size=512m", + "--env", + f"POSTGRES_PASSWORD={secrets.token_urlsafe(24)}", + "--env", + f"POSTGRES_DB={DATABASE}", + args.image, + ], + timeout=30, + ) + container_started = True + wait_until_ready(container) + + inspect = run( + [ + "docker", + "inspect", + "--format", + "{{json .HostConfig.NetworkMode}}|{{json .Mounts}}|" + '{{index .Config.Labels "livingip.canary"}}|' + '{{index .Config.Labels "livingip.canary.instance"}}', + container, + ], + ).stdout.strip() + network_mode, mounts, canary_label, instance_label = inspect.split("|", 3) + isolation = { + "network_mode": json.loads(network_mode), + "persistent_mounts": json.loads(mounts), + "canary_label": canary_label, + "instance_label": instance_label, + "temporary_data_mount": "tmpfs:/var/lib/postgresql/data", + } + receipt["isolation"] = isolation + if isolation != { + "network_mode": "none", + "persistent_mounts": [], + "canary_label": CANARY_LABEL, + "instance_label": container, + "temporary_data_mount": "tmpfs:/var/lib/postgresql/data", + }: + raise CanaryError(f"disposable container isolation mismatch: {isolation}") + + admin_psql(container, BOOTSTRAP_SQL.replace("__PARTICIPANT_HANDLE__", PARTICIPANT_HANDLE)) + admin_psql(container, APPLY_PREREQS_SQL.read_text(encoding="utf-8")) + admin_psql( + container, + f"create role {sql_identifier(IAM_USER)} login inherit password {sql_literal(password)};", + ) + migration_output = admin_psql( + container, + READ_ROLE_SQL.read_text(encoding="utf-8"), + exec_env={"OBSERVATORY_DB_IAM_USER": IAM_USER}, + ) + receipt["role_provisioning"] = parse_last_json(migration_output) + + before = fingerprint(container) + operations = operation_suite(container, password) + after = fingerprint(container) + receipt["operations"] = operations + receipt["fingerprint"] = { + "before": before, + "after": after, + "before_sha256": canonical_sha256(before), + "after_sha256": canonical_sha256(after), + "unchanged": before == after, + } + receipt["summary"] = { + "allowed_expected": sum(operation["expected"] == "allowed" for operation in operations), + "allowed_passed": sum( + operation["expected"] == "allowed" and operation["passed"] for operation in operations + ), + "denied_expected": sum(operation["expected"] == "denied" for operation in operations), + "denied_passed": sum(operation["expected"] == "denied" and operation["passed"] for operation in operations), + } + execution_passed = all(operation["passed"] for operation in operations) and before == after + except Exception as exc: + receipt["execution_error"] = str(exc)[:2000] + finally: + remove = run(["docker", "rm", "--force", container], check=False, timeout=30) + if temp_dir.exists(): + shutil.rmtree(temp_dir) + exact_name = run( + [ + "docker", + "container", + "ls", + "--all", + "--filter", + f"name=^/{container}$", + "--format", + "{{.Names}}", + ], + check=False, + timeout=15, + ) + label_scope = run( + [ + "docker", + "container", + "ls", + "--all", + "--filter", + f"label=livingip.canary={CANARY_LABEL}", + "--filter", + f"label=livingip.canary.instance={container}", + "--format", + "{{.Names}}", + ], + check=False, + timeout=15, + ) + cleanup = { + "container_started": container_started, + "remove_exit_code": remove.returncode, + "exact_name_readback_exit_code": exact_name.returncode, + "exact_name_orphans": [line for line in exact_name.stdout.splitlines() if line], + "label_readback_exit_code": label_scope.returncode, + "label_scoped_orphans": [line for line in label_scope.stdout.splitlines() if line], + "temporary_workdir_exists": temp_dir.exists(), + } + cleanup["passed"] = ( + (not container_started or remove.returncode == 0) + and exact_name.returncode == 0 + and not cleanup["exact_name_orphans"] + and label_scope.returncode == 0 + and not cleanup["label_scoped_orphans"] + and not cleanup["temporary_workdir_exists"] + ) + receipt["cleanup"] = cleanup + receipt["finished_at_utc"] = utc_now() + receipt["current_tier"] = "T2_runtime" if execution_passed and cleanup["passed"] else "T0_spec" + receipt["passed"] = execution_passed and cleanup["passed"] + + args.output.parent.mkdir(parents=True, exist_ok=True) + temporary_output = args.output.with_name(f".{args.output.name}.{run_id}.tmp") + temporary_output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary_output, args.output) + print( + json.dumps( + { + "artifact": receipt["artifact"], + "passed": receipt["passed"], + "current_tier": receipt["current_tier"], + "operations": receipt.get("summary"), + "cleanup": receipt["cleanup"], + "output": str(args.output), + }, + sort_keys=True, + ) + ) + return 0 if receipt["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/systemd/leoclean-gcp-prod-parallel-cloudsql.conf b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf new file mode 100644 index 0000000..e8768fc --- /dev/null +++ b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf @@ -0,0 +1,43 @@ +[Service] +UnsetEnvironment=PGPASSWORD +UnsetEnvironment=PGPASSFILE PGSERVICE PGHOST PGPORT PGDATABASE PGUSER PGOPTIONS +UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN +UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS CDPATH GLOBIGNORE IFS PS4 +UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT LD_DEBUG LD_DEBUG_OUTPUT LD_DYNAMIC_WEAK LD_HWCAP_MASK LD_ORIGIN_PATH LD_PROFILE LD_SHOW_AUXV LD_TRACE_LOADED_OBJECTS LD_USE_LOAD_BIAS LD_VERBOSE LD_WARN +UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT PYTHONUSERBASE PYTHONHTTPSVERIFY +UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy +UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE +UnsetEnvironment=OPENSSL_CONF OPENSSL_MODULES OPENSSL_ENGINES GCONV_PATH CREDENTIALS_DIRECTORY +UnsetEnvironment=XDG_CONFIG_HOME +UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT +ReadOnlyPaths= +ReadWritePaths= +InaccessiblePaths= +BindPaths= +BindReadOnlyPaths= +SupplementaryGroups= +ReadOnlyPaths=/home/teleo +ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state /home/teleo/.hermes/profiles/leoclean/workspace +InaccessiblePaths=-/home/teleo/.config/gcloud -/home/teleo/.pgpass -/home/teleo/.pg_service.conf -/home/teleo/.postgresql +BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin +CapabilityBoundingSet= +AmbientCapabilities= +NoNewPrivileges=yes +Group=teleo +SupplementaryGroups=teleo +Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +Environment=CLOUDSDK_CONFIG=/usr/local/libexec/livingip/leoclean-kb/gcloud-config +Environment=PGSSLMODE=verify-ca +Environment=PGSSLROOTCERT=/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem +Environment=PYTHONNOUSERSITE=1 +Environment=PYTHONSAFEPATH=1 +Environment=TELEO_KB_MODE=cloudsql +Environment=TELEO_GCP_METADATA_ONLY=1 +Environment=TELEO_GCP_PROJECT=teleo-501523 +Environment=TELEO_CLOUDSQL_HOST=10.61.0.3 +Environment=TELEO_CLOUDSQL_PORT=5432 +Environment=TELEO_CLOUDSQL_DB=teleo_canonical +Environment=TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical +Environment=TELEO_CLOUDSQL_USER=leoclean_kb_runtime +Environment=TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password +Environment=TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 diff --git a/tests/test_gcp_generated_db_direct_claim_suite.py b/tests/test_gcp_generated_db_direct_claim_suite.py index 5aa40ea..ca18c64 100644 --- a/tests/test_gcp_generated_db_direct_claim_suite.py +++ b/tests/test_gcp_generated_db_direct_claim_suite.py @@ -109,13 +109,17 @@ def test_wrapper_binds_private_read_only_database_and_records_calls(tmp_path: Pa host="10.61.0.3", project="teleo-501523", password_secret="test-secret-name", + ssl_root_cert=tmp_path / "cloudsql-server-ca.pem", run_nonce="nonce", ) assert "dbname=$TARGET_DATABASE" in wrapper assert '--db "$TARGET_DATABASE"' in wrapper assert '--canonical-db "$TARGET_DATABASE"' in wrapper - assert "sslmode=require" in wrapper + assert "--credential-mode clone-readonly" in wrapper + assert "--user postgres" in wrapper + assert "sslmode=verify-ca" in wrapper + assert "sslrootcert=$SSL_ROOT_CERT" in wrapper assert "begin transaction read only" in wrapper assert 'export PGOPTIONS="-c default_transaction_read_only=on"' in wrapper assert "default_transaction_read_only" in wrapper @@ -138,6 +142,7 @@ def test_database_identity_requires_private_tls_and_read_only_target() -> None: "default_transaction_read_only": "on", }, "teleo_clone_test", + "1234", ) with pytest.raises(RuntimeError, match="not RFC1918-private"): @@ -151,6 +156,7 @@ def test_database_identity_requires_private_tls_and_read_only_target() -> None: "default_transaction_read_only": "on", }, "teleo_clone_test", + "1234", ) @@ -182,6 +188,7 @@ def test_database_fingerprint_uses_full_normalized_manifest(monkeypatch: pytest. "host": "10.61.0.3", "target_db": "teleo_clone_test", "manifest_sql": manifest_sql, + "ssl_root_cert": tmp_path / "cloudsql-server-ca.pem", }, )() output = "\n".join( @@ -280,6 +287,7 @@ def test_generated_wrapper_executes_only_clone_bound_default_read_only_tool(tmp_ host="10.61.0.3", project="teleo-501523", password_secret="test-secret", + ssl_root_cert=tmp_path / "cloudsql-server-ca.pem", run_nonce="test-nonce", system_exec_path=f"{fake_bin}:/usr/bin:/bin", ) @@ -306,6 +314,8 @@ def test_generated_wrapper_executes_only_clone_bound_default_read_only_tool(tmp_ "--canonical-db", "teleo_clone_test", ] + assert tool_receipt["argv"][tool_receipt["argv"].index("--credential-mode") + 1] == "clone-readonly" + assert tool_receipt["argv"][tool_receipt["argv"].index("--user") + 1] == "postgres" events = [json.loads(line) for line in log.read_text().splitlines()] assert [event["phase"] for event in events] == ["start", "end"] assert events[0]["database_identity"]["default_transaction_read_only"] == "on" diff --git a/tests/test_gcp_generated_db_working_leo_suite.py b/tests/test_gcp_generated_db_working_leo_suite.py index a3e5526..f4543ac 100644 --- a/tests/test_gcp_generated_db_working_leo_suite.py +++ b/tests/test_gcp_generated_db_working_leo_suite.py @@ -95,9 +95,7 @@ def source_baseline_payload() -> dict[str, object]: } -def capability_receipt( - role: str, target_db: str = "teleo_clone_working_leo", *, writable: bool | None = None -) -> str: +def capability_receipt(role: str, target_db: str = "teleo_clone_working_leo", *, writable: bool | None = None) -> str: capabilities = { "kb_read": (False, False, False, False, False), "kb_stage_writer": (True, False, False, False, False), @@ -131,6 +129,7 @@ def executor_args(**overrides: object) -> argparse.Namespace: "project": "teleo-501523", "instance": "teleo-pgvector-standby", "password_secret": "read-secret", + "ssl_root_cert": Path("ops/gcp-teleo-pgvector-standby-server-ca.pem").resolve(), "read_role": "kb_read", "stage_role": "kb_stage_writer", "review_role": "kb_review", @@ -229,6 +228,10 @@ def test_executor_binds_every_call_and_defaults_read_only_except_controller_phas assert len(calls) == 4 assert all("dbname=teleo_clone_working_leo" in call["command"][1] for call in calls) + assert all("sslmode=verify-ca" in call["command"][1] for call in calls) + assert all("sslrootcert=" in call["command"][1] for call in calls) + assert all(call["env"]["PGSSLMODE"] == "verify-ca" for call in calls) + assert all(call["env"]["PGSSLROOTCERT"].endswith("gcp-teleo-pgvector-standby-server-ca.pem") for call in calls) assert all("expected_database=teleo_clone_working_leo" in call["command"] for call in calls) assert calls[0]["env"]["PGOPTIONS"] == "-c default_transaction_read_only=on" assert [call["env"]["PGOPTIONS"] for call in calls[1:]] == [ @@ -332,14 +335,18 @@ def test_source_snapshots_are_receipt_backed_and_inherited_source_checks_are_not } adapter = suite.RuntimeAdapter(args, suite.CloudSqlExecutor(args), source, {"system_identifier": "target"}) - assert adapter._guard_snapshot( - suite.bound.PRODUCTION_CONTAINER, - suite.bound.PRODUCTION_DB, - tables=suite.composition.PRODUCTION_TABLES, - ) == source_payload["guard_snapshot"] - assert adapter._gate_schema(suite.bound.PRODUCTION_CONTAINER, suite.bound.PRODUCTION_DB) == source_payload[ - "gate_schema" - ] + assert ( + adapter._guard_snapshot( + suite.bound.PRODUCTION_CONTAINER, + suite.bound.PRODUCTION_DB, + tables=suite.composition.PRODUCTION_TABLES, + ) + == source_payload["guard_snapshot"] + ) + assert ( + adapter._gate_schema(suite.bound.PRODUCTION_CONTAINER, suite.bound.PRODUCTION_DB) + == source_payload["gate_schema"] + ) contract = suite.composition_check_contract() required = [name for name in contract if name not in suite.UNSUPPORTED_INHERITED_COMPOSITION_CHECKS] assert len(contract) == 34 @@ -519,7 +526,7 @@ def test_lifecycle_wrapper_exposes_only_read_or_exact_stage_and_never_delivery(t assert "accepts no model-supplied arguments" in wrapper assert "--stage-password-secret" in wrapper assert "--read-role" in wrapper - assert 'user=$READ_ROLE' in wrapper + assert "user=$READ_ROLE" in wrapper assert "default_transaction_read_only=on" in wrapper assert "systemctl restart" not in wrapper assert "send_message" not in wrapper @@ -535,6 +542,9 @@ def test_lifecycle_wrapper_exposes_only_read_or_exact_stage_and_never_delivery(t ) assert catalog_wrapper.count('"prompt_id": "DC-01"') == 2 assert '--user "$READ_ROLE"' in catalog_wrapper + assert "sslmode=verify-ca" in catalog_wrapper + assert "sslrootcert=$SSL_ROOT_CERT" in catalog_wrapper + assert "sslmode=require" not in catalog_wrapper def test_capability_receipt_rejects_superuser_or_wrong_phase_acl() -> None: diff --git a/tests/test_gcp_leoclean_runtime_permissions.py b/tests/test_gcp_leoclean_runtime_permissions.py new file mode 100644 index 0000000..dee0eed --- /dev/null +++ b/tests/test_gcp_leoclean_runtime_permissions.py @@ -0,0 +1,791 @@ +import hashlib +import json +import os +import stat +import subprocess +from collections.abc import Mapping +from pathlib import Path + +import pytest + +from ops import verify_gcp_leoclean_runtime_permissions as verifier + +RUN_ID = "20260715-leo-security" +SOURCE_REF = f"leo-runtime-permission:{RUN_ID}" +SECRET_VALUE = "unit-only-secret-value%42" +AGENT_ID = "11111111-1111-4111-8111-111111111111" + + +class FakeRunner: + def __init__( + self, + *, + sqlstate_override: tuple[str, str] | None = None, + sqlstate_missing_for: str | None = None, + unexpected_success_for: str | None = None, + scoped_failure: bool = False, + administrator_mode: str = "denied", + identity_override: Mapping[str, object] | None = None, + role_posture_override: Mapping[str, object] | None = None, + stage_owner_role_posture_override: Mapping[str, object] | None = None, + function_posture_override: Mapping[str, Mapping[str, object]] | None = None, + stage_definition_override: Mapping[str, object] | None = None, + catalog_posture_override: Mapping[str, object] | None = None, + ) -> None: + self.sqlstate_override = sqlstate_override + self.sqlstate_missing_for = sqlstate_missing_for + self.unexpected_success_for = unexpected_success_for + self.scoped_failure = scoped_failure + self.administrator_mode = administrator_mode + self.identity_override = dict(identity_override or {}) + self.role_posture_override = dict(role_posture_override or {}) + self.stage_owner_role_posture_override = dict(stage_owner_role_posture_override or {}) + self.function_posture_override = { + name: dict(values) for name, values in (function_posture_override or {}).items() + } + self.stage_definition_override = dict(stage_definition_override or {}) + self.catalog_posture_override = dict(catalog_posture_override or {}) + self.calls: list[tuple[list[str], dict[str, str], int, bool]] = [] + self.negative = {check.sql: check for check in verifier._negative_checks(RUN_ID, SOURCE_REF)} + + def __call__( + self, + command: list[str], + env: Mapping[str, str], + timeout: int, + discard_stdout: bool, + ) -> verifier.CommandResult: + self.calls.append((list(command), dict(env), timeout, discard_stdout)) + + if command[0] == verifier.GCLOUD_BIN: + scoped = f"--secret={verifier.SCOPED_PASSWORD_SECRET}" in command + if scoped: + if self.scoped_failure: + return verifier.CommandResult(1, SECRET_VALUE, f"scoped failure {SECRET_VALUE}") + return verifier.CommandResult(0, f"{SECRET_VALUE}\n", f"ignored warning {SECRET_VALUE}") + assert f"--secret={verifier.ADMINISTRATOR_PASSWORD_SECRET}" in command + assert discard_stdout is True + if self.administrator_mode == "success": + return verifier.CommandResult(0, SECRET_VALUE, f"ignored {SECRET_VALUE}") + if self.administrator_mode == "wrong-error": + return verifier.CommandResult(1, SECRET_VALUE, f"network unavailable {SECRET_VALUE}") + return verifier.CommandResult( + 1, + SECRET_VALUE, + ( + "ERROR: PERMISSION_DENIED: Permission " + f"'{verifier.IAM_PERMISSION}' denied for the administrator secret; {SECRET_VALUE}" + ), + ) + + assert command[0] == verifier.PSQL_BIN + assert discard_stdout is False + sql = next(part.removeprefix("--command=") for part in command if part.startswith("--command=")) + + if sql == verifier._identity_sql(): + identity: dict[str, object] = { + "current_user": verifier.RUNTIME_DATABASE_ROLE, + "database": verifier.CANONICAL_DATABASE, + "leak": SECRET_VALUE, + "server_addr": verifier.PRIVATE_CLOUDSQL_HOST, + "server_port": verifier.PRIVATE_CLOUDSQL_PORT, + "session_user": verifier.RUNTIME_DATABASE_ROLE, + "ssl": True, + "ssl_version": "TLSv1.3", + "system_identifier": verifier.EXPECTED_SYSTEM_IDENTIFIER, + } + identity.update(self.identity_override) + return verifier.CommandResult(0, json.dumps(identity), f"ignored {SECRET_VALUE}") + if sql == verifier._allowed_read_sql(): + return verifier.CommandResult( + 0, + json.dumps( + { + "claims_rows_sampled": 1, + "leak": SECRET_VALUE, + "proposal_rows_sampled": 0, + } + ), + f"ignored {SECRET_VALUE}", + ) + if sql == verifier._role_posture_sql(): + role_posture: dict[str, object] = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": True, + "can_replicate": False, + "connection_limit": 8, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "leak": SECRET_VALUE, + "role": verifier.RUNTIME_DATABASE_ROLE, + } + role_posture.update(self.role_posture_override) + return verifier.CommandResult(0, json.dumps(role_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._stage_owner_role_posture_sql(): + owner_posture: dict[str, object] = { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": False, + "can_replicate": False, + "connection_limit": -1, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "leak": SECRET_VALUE, + "role": verifier.STAGE_OWNER_DATABASE_ROLE, + } + owner_posture.update(self.stage_owner_role_posture_override) + return verifier.CommandResult(0, json.dumps(owner_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._function_privilege_posture_sql(): + function_posture: dict[str, dict[str, object]] = { + name: { + "execute": expected_execute, + "exists": expected_exists, + "leak": SECRET_VALUE, + } + for name, _signature, expected_exists, expected_execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS + } + for name, override in self.function_posture_override.items(): + function_posture.setdefault(name, {}).update(override) + function_posture["unexpected_leak"] = {"value": SECRET_VALUE} + return verifier.CommandResult(0, json.dumps(function_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._stage_function_definition_sql(): + stage_definition: dict[str, object] = { + "acl_exact": True, + "argument_modes": None, + "configuration": ["search_path=pg_catalog, pg_temp"], + "exists": True, + "leak": SECRET_VALUE, + "runtime_execute": True, + "source": verifier.EXPECTED_STAGE_FUNCTION_SOURCE, + **dict(verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS), + } + stage_definition.update(self.stage_definition_override) + return verifier.CommandResult(0, json.dumps(stage_definition), f"ignored {SECRET_VALUE}") + if sql == verifier._catalog_privilege_posture_sql(): + catalog_posture: dict[str, object] = {field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS} + catalog_posture.update( + { + "leak": SECRET_VALUE, + **{field: True for field in verifier.CATALOG_TRUE_FIELDS}, + } + ) + catalog_posture.update(self.catalog_posture_override) + return verifier.CommandResult(0, json.dumps(catalog_posture), f"ignored {SECRET_VALUE}") + if sql == verifier._canary_count_sql(SOURCE_REF): + return verifier.CommandResult(0, "0\n", f"ignored {SECRET_VALUE}") + if sql == verifier._stage_sql(RUN_ID, SOURCE_REF): + return verifier.CommandResult( + 0, + json.dumps( + { + "agent_id_matches": True, + "proposal": { + "leak": SECRET_VALUE, + "proposed_by_agent_id": AGENT_ID, + "proposed_by_handle": "leo", + "source_ref": SOURCE_REF, + "status": "pending_review", + }, + } + ), + f"ignored {SECRET_VALUE}", + ) + + check = self.negative[sql] + if check.name == self.unexpected_success_for: + return verifier.CommandResult(0, SECRET_VALUE, f"unexpected success {SECRET_VALUE}") + if check.name == self.sqlstate_missing_for: + return verifier.CommandResult(1, SECRET_VALUE, f"denied without state {SECRET_VALUE}") + if check.expected_connection_denial: + return verifier.CommandResult( + 2, + SECRET_VALUE, + ( + f'connection failed: FATAL: permission denied for database "{check.database}"\n' + f"DETAIL: User does not have CONNECT privilege. {SECRET_VALUE}" + ), + ) + observed = check.expected_sqlstate + if self.sqlstate_override is not None and check.name == self.sqlstate_override[0]: + observed = self.sqlstate_override[1] + return verifier.CommandResult( + 1, + SECRET_VALUE, + f"ERROR: {observed}: expected test denial; {SECRET_VALUE}\nLOCATION: test", + ) + + +def poisoned_environment() -> dict[str, str]: + return { + "CLOUDSDK_AUTH_ACCESS_TOKEN": "must-not-reach-gcloud", + "GOOGLE_APPLICATION_CREDENTIALS": "/tmp/must-not-reach-gcloud.json", + "HOME": "/home/teleo", + "LANG": "C.UTF-8", + "LD_PRELOAD": "/tmp/must-not-load.so", + "OTHER": "preserved", + "PGDATABASE": "wrong-db", + "PGHOST": "public.example.invalid", + "PGPASSWORD": "administrator-password", + "PGSERVICE": "broad-service", + "pgsslmode": "disable", + } + + +def run_success(runner: FakeRunner) -> dict[str, object]: + return verifier.verify_runtime_permissions( + RUN_ID, + runner=runner, + base_env=poisoned_environment(), + effective_user="teleo", + dependency_validator=lambda: None, + ) + + +def assert_child_boundaries(runner: FakeRunner) -> None: + assert runner.calls + for command, env, timeout, discard_stdout in runner.calls: + assert SECRET_VALUE not in "\n".join(command) + assert timeout == verifier.COMMAND_TIMEOUT_SECONDS + pg_keys = sorted(key for key in env if key.upper().startswith("PG")) + if command[0] == verifier.PSQL_BIN: + assert pg_keys == ["PGPASSWORD", "PGSSLMODE", "PGSSLROOTCERT"] + assert env["PGPASSWORD"] == SECRET_VALUE + assert env["PGSSLMODE"] == "verify-ca" + assert env["PGSSLROOTCERT"] == str(verifier.SERVER_CA_PATH) + assert discard_stdout is False + else: + assert pg_keys == [] + assert "CLOUDSDK_AUTH_ACCESS_TOKEN" not in env + assert "GOOGLE_APPLICATION_CREDENTIALS" not in env + assert env["HOME"] == "/home/teleo" + assert env["LANG"] == "C" + assert env["LC_ALL"] == "C" + assert env["PATH"] == verifier.CHILD_PATH + assert "LD_PRELOAD" not in env + assert "OTHER" not in env + + +def test_live_receipt_contract_is_sanitized_and_uses_exact_child_environments() -> None: + runner = FakeRunner() + + receipt = run_success(runner) + serialized = verifier.canonical_json(receipt) + + assert receipt["status"] == "pass" + assert receipt["current_tier"] == verifier.REQUIRED_TIER + assert receipt["database_identity"] == { + "current_user": verifier.RUNTIME_DATABASE_ROLE, + "database": verifier.CANONICAL_DATABASE, + "server_addr": verifier.PRIVATE_CLOUDSQL_HOST, + "server_port": verifier.PRIVATE_CLOUDSQL_PORT, + "session_user": verifier.RUNTIME_DATABASE_ROLE, + "ssl": True, + "ssl_version": "TLSv1.3", + "system_identifier": verifier.EXPECTED_SYSTEM_IDENTIFIER, + } + assert receipt["checks"]["rolled_back_proposal"] == { + "agent_id_matches": True, + "proposed_by_agent_id": AGENT_ID, + "proposed_by_handle": "leo", + "source_ref": SOURCE_REF, + "status": "pending_review", + "transaction": "rolled_back", + } + assert receipt["checks"]["canary_rows_before"] == 0 + assert receipt["checks"]["canary_rows_after"] == 0 + assert receipt["checks"]["role_posture"] == { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": True, + "can_replicate": False, + "connection_limit": 8, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "role": verifier.RUNTIME_DATABASE_ROLE, + } + assert receipt["checks"]["stage_owner_role_posture"] == { + "bypasses_rls": False, + "can_create_db": False, + "can_create_role": False, + "can_login": False, + "can_replicate": False, + "connection_limit": -1, + "direct_membership_edges": 0, + "inherits_privileges": False, + "is_superuser": False, + "role": verifier.STAGE_OWNER_DATABASE_ROLE, + } + assert receipt["checks"]["function_privileges"] == { + name: { + "execute": expected_execute, + "exists": expected_exists, + "signature": signature, + } + for name, signature, expected_exists, expected_execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS + } + assert receipt["checks"]["catalog_privileges"] == { + **{field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS}, + **{field: True for field in verifier.CATALOG_TRUE_FIELDS}, + } + assert receipt["checks"]["stage_function_definition"] == { + "acl_exact": True, + "argument_modes": None, + "configuration": ["search_path=pg_catalog, pg_temp"], + "exists": True, + **dict(verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS), + "runtime_execute": True, + "signature": verifier.STAGE_FUNCTION_SIGNATURE, + "source_sha256": verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256, + } + assert len(receipt["checks"]["negative_permissions"]) == len(verifier._negative_checks(RUN_ID, SOURCE_REF)) + assert receipt["secret_access"]["administrator"]["classification"] == "iam_permission_denied" + assert receipt["secret_access"]["administrator"]["stdout_discarded"] is True + assert SECRET_VALUE not in serialized + assert "administrator-password" not in serialized + assert "stage_leoclean_proposal is restricted" not in serialized + assert_child_boundaries(runner) + + administrator_call = runner.calls[-1] + assert administrator_call[0][0] == verifier.GCLOUD_BIN + assert administrator_call[3] is True + + +@pytest.mark.parametrize( + ("attribute", "unsafe_value"), + [ + ("can_login", False), + ("is_superuser", True), + ("can_create_db", True), + ("can_create_role", True), + ("inherits_privileges", True), + ("can_replicate", True), + ("bypasses_rls", True), + ("connection_limit", 7), + ], +) +def test_any_unsafe_runtime_role_attribute_fails_closed(attribute: str, unsafe_value: object) -> None: + runner = FakeRunner(role_posture_override={attribute: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "runtime_role_posture_mismatch" + assert caught.value.check == "runtime_role_posture" + assert SECRET_VALUE not in str(caught.value) + + +def test_any_direct_runtime_role_membership_edge_fails_closed() -> None: + runner = FakeRunner(role_posture_override={"direct_membership_edges": 1}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "runtime_role_posture_mismatch" + assert caught.value.check == "runtime_role_posture" + + +@pytest.mark.parametrize( + ("attribute", "unsafe_value"), + [ + ("role", "unexpected-owner"), + ("can_login", True), + ("is_superuser", True), + ("can_create_db", True), + ("can_create_role", True), + ("inherits_privileges", True), + ("can_replicate", True), + ("bypasses_rls", True), + ("connection_limit", 0), + ("direct_membership_edges", 1), + ], +) +def test_any_unsafe_stage_owner_role_posture_fails_closed(attribute: str, unsafe_value: object) -> None: + runner = FakeRunner(stage_owner_role_posture_override={attribute: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_owner_role_posture_mismatch" + assert caught.value.check == "stage_owner_role_posture" + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize( + "function_name", + ["approve_strict_proposal", "assert_approved_proposal", "finish_approved_proposal"], +) +def test_reviewer_or_apply_execute_grant_fails_closed(function_name: str) -> None: + runner = FakeRunner(function_posture_override={function_name: {"execute": True}}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "function_privilege_posture_mismatch" + assert caught.value.check == "function_privilege_posture" + + +@pytest.mark.parametrize( + "function_name", + [ + "stage_leoclean_proposal_5_arg", + "approve_strict_proposal", + "assert_approved_proposal", + "finish_approved_proposal", + ], +) +def test_missing_expected_exact_function_fails_closed(function_name: str) -> None: + runner = FakeRunner(function_posture_override={function_name: {"execute": False, "exists": False}}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "function_privilege_posture_mismatch" + assert caught.value.check == "function_privilege_posture" + + +def test_forged_overload_presence_or_missing_stage_execute_fails_closed() -> None: + forged = FakeRunner(function_posture_override={"stage_leoclean_proposal_6_arg": {"exists": True}}) + missing_execute = FakeRunner(function_posture_override={"stage_leoclean_proposal_5_arg": {"execute": False}}) + + for runner in (forged, missing_execute): + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + assert caught.value.code == "function_privilege_posture_mismatch" + + +def _drifted_value(expected: object) -> object: + if isinstance(expected, bool): + return not expected + if isinstance(expected, int): + return expected + 1 + if isinstance(expected, str): + return f"{expected}-drift" + if isinstance(expected, list): + return [*expected, "drift"] + raise AssertionError(f"unsupported test expectation type: {type(expected).__name__}") + + +@pytest.mark.parametrize( + ("field", "unsafe_value"), + [(field, _drifted_value(expected)) for field, expected in verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS], +) +def test_any_stage_function_metadata_drift_fails_closed(field: str, unsafe_value: object) -> None: + runner = FakeRunner(stage_definition_override={field: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_function_definition_mismatch" + assert caught.value.check == "stage_function_definition" + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize( + ("field", "unsafe_value"), + [ + ("exists", False), + ("argument_modes", ["i", "i", "i", "i", "i"]), + ("configuration", ["search_path=public"]), + ("acl_exact", False), + ("runtime_execute", False), + ("leakproof", 0), + ("argument_defaults", False), + ], +) +def test_any_stage_function_contract_drift_fails_closed(field: str, unsafe_value: object) -> None: + runner = FakeRunner(stage_definition_override={field: unsafe_value}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_function_definition_mismatch" + assert caught.value.check == "stage_function_definition" + + +@pytest.mark.parametrize( + "unsafe_source", + [ + None, + 42, + verifier.EXPECTED_STAGE_FUNCTION_SOURCE.replace("return to_jsonb(staged);", "return '{}'::jsonb;"), + "x" * (verifier.MAX_STAGE_FUNCTION_SOURCE_BYTES + 1), + ], +) +def test_missing_malformed_oversized_or_semantically_drifted_stage_body_fails_closed( + unsafe_source: object, +) -> None: + runner = FakeRunner(stage_definition_override={"source": unsafe_source}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "stage_function_definition_mismatch" + assert caught.value.check == "stage_function_definition" + failure = verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value)) + assert "stage_leoclean_proposal is restricted" not in failure + assert SECRET_VALUE not in failure + + +def test_stage_body_normalization_accepts_only_line_ending_and_framing_newline_changes() -> None: + crlf_source = "\r\n" + verifier.EXPECTED_STAGE_FUNCTION_SOURCE.strip("\n").replace("\n", "\r\n") + "\r\n" + + receipt = run_success(FakeRunner(stage_definition_override={"source": crlf_source})) + + assert ( + receipt["checks"]["stage_function_definition"]["source_sha256"] + == verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256 + ) + + +def test_embedded_stage_body_attestation_is_tied_to_the_provisioning_sql_source() -> None: + provisioning_sql = (Path(__file__).resolve().parents[1] / "ops" / "gcp_leoclean_runtime_role.sql").read_text( + encoding="utf-8" + ) + function_anchor = "create or replace function kb_stage.stage_leoclean_proposal(" + assert provisioning_sql.count(function_anchor) == 1 + function_section = provisioning_sql.split(function_anchor, 1)[1] + assert function_section.count("as $function$") == 1 + provisioned_source = function_section.split("as $function$", 1)[1].split("$function$;", 1)[0] + + assert provisioned_source == verifier.EXPECTED_STAGE_FUNCTION_SOURCE + assert ( + hashlib.sha256(verifier.normalize_stage_function_source(provisioned_source).encode("utf-8")).hexdigest() + == verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256 + ) + + +def test_catalog_queries_use_exact_regprocedure_signatures_and_both_membership_directions() -> None: + function_sql = verifier._function_privilege_posture_sql() + role_sql = verifier._role_posture_sql() + owner_role_sql = verifier._stage_owner_role_posture_sql() + stage_definition_sql = verifier._stage_function_definition_sql() + catalog_sql = verifier._catalog_privilege_posture_sql() + + assert "pg_catalog.to_regprocedure(signature)" in function_sql + for _name, signature, _exists, _execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS: + assert signature in function_sql + assert "membership.roleid = role_row.oid" in role_sql + assert "membership.member = role_row.oid" in role_sql + assert "membership.roleid = role_row.oid" in owner_role_sql + assert "membership.member = role_row.oid" in owner_role_sql + assert verifier.STAGE_OWNER_DATABASE_ROLE in owner_role_sql + assert verifier.STAGE_FUNCTION_SIGNATURE in stage_definition_sql + assert "left join pg_catalog.pg_proc" in stage_definition_sql + assert "function_catalog.prosrc" in stage_definition_sql + assert "function_catalog.prokind" in stage_definition_sql + assert "function_catalog.proargtypes" in stage_definition_sql + assert "function_catalog.prosecdef" in stage_definition_sql + assert "pg_catalog.pg_get_function_result" in stage_definition_sql + assert "pg_catalog.aclexplode" in stage_definition_sql + assert "pg_catalog.pg_shdepend" in catalog_sql + assert "pg_catalog.has_table_privilege" in catalog_sql + assert "pg_catalog.has_column_privilege" in catalog_sql + assert "pg_catalog.has_sequence_privilege" in catalog_sql + assert "pg_catalog.has_function_privilege" in catalog_sql + assert "pg_catalog.aclexplode" in catalog_sql + assert "nspname !~ '^pg_'" in catalog_sql + assert "nspname <> 'information_schema'" in catalog_sql + assert verifier.STAGE_OWNER_DATABASE_ROLE in catalog_sql + assert "public_database_connect_grants" in catalog_sql + assert "public_large_object_mutation_routine_execute" in catalog_sql + assert "'lo_creat'" in catalog_sql + assert "'lo_open'" in catalog_sql + for schema, relation in verifier.READ_TABLE_ALLOWLIST: + assert schema in catalog_sql + assert relation in catalog_sql + + +@pytest.mark.parametrize("field", verifier.CATALOG_ZERO_COUNT_FIELDS) +def test_any_unexpected_catalog_privilege_fails_closed(field: str) -> None: + runner = FakeRunner(catalog_posture_override={field: 1}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "catalog_privilege_posture_mismatch" + assert caught.value.check == "catalog_privilege_posture" + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize("field", verifier.CATALOG_TRUE_FIELDS) +def test_any_missing_required_catalog_privilege_fails_closed(field: str) -> None: + runner = FakeRunner(catalog_posture_override={field: False}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "catalog_privilege_posture_mismatch" + + +def test_legacy_and_template_database_connect_are_both_behaviorally_denied() -> None: + runner = FakeRunner() + + receipt = run_success(runner) + + denials = {check["check"]: check for check in receipt["checks"]["negative_permissions"]} + for name in ("legacy_database_connect", "template_database_connect"): + assert denials[name] == { + "check": name, + "expected_sqlstate": "42501", + "observed_error_class": "connect_privilege_denied", + "observed_sqlstate": "not_exposed_by_libpq_startup", + "result": "denied", + } + database_calls = [ + " ".join(command) for command, _env, _timeout, _discard in runner.calls if command[0] == verifier.PSQL_BIN + ] + assert any(f"dbname={verifier.LEGACY_DATABASE}" in command for command in database_calls) + assert any(f"dbname={verifier.TEMPLATE_DATABASE}" in command for command in database_calls) + + +def test_required_sqlstate_mismatch_fails_closed_without_secret_in_error() -> None: + runner = FakeRunner(sqlstate_override=("canonical_insert", "23505")) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + error = caught.value + assert error.code == "negative_permission_sqlstate_mismatch" + assert error.check == "canonical_insert" + assert error.expected_sqlstate == "42501" + assert error.observed_sqlstate == "23505" + assert SECRET_VALUE not in str(error) + assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, error)) + assert_child_boundaries(runner) + + +def test_missing_verbose_sqlstate_fails_closed_without_raw_stderr() -> None: + runner = FakeRunner(sqlstate_missing_for="forged_proposer_overload") + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + error = caught.value + assert error.code == "negative_permission_sqlstate_missing" + assert error.expected_sqlstate == "42883" + assert error.observed_sqlstate == "missing" + assert SECRET_VALUE not in json.dumps(error.as_dict()) + + +def test_unclassified_database_startup_failure_cannot_masquerade_as_connect_denial() -> None: + runner = FakeRunner(sqlstate_missing_for="legacy_database_connect") + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + error = caught.value + assert error.code == "negative_connection_denial_mismatch" + assert error.check == "legacy_database_connect" + assert error.observed_sqlstate == "startup_error_unclassified" + assert SECRET_VALUE not in json.dumps(error.as_dict()) + + +def test_unexpected_permission_success_fails_closed_and_transaction_sql_has_rollback() -> None: + runner = FakeRunner(unexpected_success_for="proposal_update") + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "negative_permission_unexpectedly_succeeded" + assert caught.value.observed_sqlstate == "success" + proposal_update = next( + check for check in verifier._negative_checks(RUN_ID, SOURCE_REF) if check.name == "proposal_update" + ) + assert proposal_update.sql.startswith("begin;\n") + assert proposal_update.sql.endswith("\nrollback;") + assert SECRET_VALUE not in str(caught.value) + + +@pytest.mark.parametrize("administrator_mode", ["success", "wrong-error"]) +def test_administrator_secret_must_be_exact_iam_denial_and_stdout_is_discarded(administrator_mode: str) -> None: + runner = FakeRunner(administrator_mode=administrator_mode) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + expected_code = ( + "administrator_secret_unexpectedly_readable" + if administrator_mode == "success" + else "administrator_secret_denial_not_iam_permission" + ) + assert caught.value.code == expected_code + assert SECRET_VALUE not in str(caught.value) + assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value)) + assert runner.calls[-1][3] is True + + +def test_scoped_secret_failure_never_repeats_command_output() -> None: + runner = FakeRunner(scoped_failure=True) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "scoped_secret_access_failed" + assert SECRET_VALUE not in str(caught.value) + assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value)) + assert len(runner.calls) == 1 + + +@pytest.mark.parametrize( + "invalid", + ["short", "UPPERCASE-RUN", "contains_underscore", "contains.dot", " leading-space", "a" * 65], +) +def test_run_id_validation_is_strict(invalid: str) -> None: + with pytest.raises(ValueError, match="8-64 lowercase"): + verifier.validate_run_id(invalid) + + +def test_wrong_identity_or_non_ssl_connection_fails_before_permission_probes() -> None: + runner = FakeRunner(identity_override={"ssl": False}) + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + assert caught.value.code == "private_ssl_identity_mismatch" + assert len(runner.calls) == 2 + assert SECRET_VALUE not in str(caught.value) + + +def test_optional_receipt_is_canonical_mode_0600_and_refuses_symlink(tmp_path: Path) -> None: + payload = {"z": 2, "a": {"safe": True}} + output = tmp_path / "receipt.json" + + verifier.write_receipt(output, payload) + + assert output.read_text(encoding="utf-8") == verifier.canonical_json(payload) + assert stat.S_IMODE(output.stat().st_mode) == 0o600 + assert output.read_text(encoding="utf-8").startswith('{"a":') + + target = tmp_path / "target.json" + target.write_text("untouched", encoding="utf-8") + symlink = tmp_path / "symlink.json" + symlink.symlink_to(target) + with pytest.raises(OSError): + verifier.write_receipt(symlink, payload) + assert target.read_text(encoding="utf-8") == "untouched" + + +def test_subprocess_runner_discards_administrator_stdout_at_file_descriptor(monkeypatch: pytest.MonkeyPatch) -> None: + observed: dict[str, object] = {} + + def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + observed["command"] = command + observed.update(kwargs) + return subprocess.CompletedProcess(command, 1, stdout=None, stderr="PERMISSION_DENIED") + + monkeypatch.setattr(verifier.subprocess, "run", fake_run) + + result = verifier.subprocess_runner([verifier.GCLOUD_BIN, "test"], os.environ, 3, True) + + assert observed["stdout"] is subprocess.DEVNULL + assert observed["stdin"] is subprocess.DEVNULL + assert observed["stderr"] is subprocess.PIPE + assert result.stdout == "" + assert result.returncode == 1 diff --git a/tests/test_gcp_leoclean_service_environment.py b/tests/test_gcp_leoclean_service_environment.py new file mode 100644 index 0000000..57c37ae --- /dev/null +++ b/tests/test_gcp_leoclean_service_environment.py @@ -0,0 +1,309 @@ +import json +import os +import subprocess +import sys +import time +from collections.abc import Mapping +from pathlib import Path + +import pytest + +from ops import verify_gcp_leoclean_service_environment as verifier + + +def environment_block(environment: Mapping[str, str]) -> bytes: + return b"\0".join(f"{key}={value}".encode() for key, value in environment.items()) + b"\0" + + +def wait_for_process_environment( + process: subprocess.Popen[bytes], + expected: Mapping[str, str], + *, + timeout_seconds: float = 5.0, +) -> None: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if process.poll() is not None: + pytest.fail("environment fixture process exited before becoming readable") + try: + observed = verifier.parse_environment(Path(f"/proc/{process.pid}/environ").read_bytes()) + except (OSError, verifier.VerificationError): + time.sleep(0.01) + continue + if observed == expected: + return + time.sleep(0.01) + pytest.fail("environment fixture process did not become readable") + + +def test_all_expected_fields_pass_and_receipt_contains_no_values() -> None: + environment = verifier.expected_environment() + environment.update({"HOME": "/home/teleo", "LANG": "C.UTF-8"}) + + parsed = verifier.parse_environment(environment_block(environment)) + validated = verifier.validate_environment(parsed) + receipt = { + "runtime_environment": "scoped", + "status": "pass", + "validated_fields": list(validated), + } + rendered = verifier.canonical_json(receipt) + + assert validated == tuple(sorted(verifier.expected_environment())) + assert json.loads(rendered) == receipt + assert rendered == json.dumps(receipt, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n" + for value in verifier.expected_environment().values(): + assert value not in rendered + + +@pytest.mark.parametrize("field", sorted(verifier.expected_environment())) +def test_each_expected_field_mismatch_fails_closed(field: str) -> None: + environment = verifier.expected_environment() + environment[field] = "wrong-value-that-is-not-a-secret" + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + assert caught.value.code == "environment_mismatch" + assert caught.value.fields == (field,) + + +@pytest.mark.parametrize( + ("field", "unsafe_value", "expected_code"), + [ + ("TELEO_CLOUDSQL_USER", "postgres", "environment_mismatch"), + ( + "TELEO_CLOUDSQL_PASSWORD_SECRET", + verifier.ADMINISTRATOR_PASSWORD_SECRET, + "administrator_secret_reference", + ), + ], +) +def test_later_dropin_effective_override_fails_without_outputting_value( + field: str, + unsafe_value: str, + expected_code: str, +) -> None: + environment = verifier.expected_environment() + environment[field] = unsafe_value + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert caught.value.code == expected_code + assert unsafe_value not in rendered + + +@pytest.mark.parametrize( + ("field", "safe_label"), + [ + ("PGPASSWORD", "PG*"), + ("pgservice", "PG*"), + ("CLOUDSDK_CORE_PROJECT", "CLOUDSDK_*"), + ("CLOUDSDK_AUTH_ACCESS_TOKEN", "CLOUDSDK_*"), + ("cloudsdk_auth_refresh_token", "CLOUDSDK_*"), + ("GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_APPLICATION_CREDENTIALS"), + ("LD_PRELOAD", "LD_*"), + ("LD_LIBRARY_PATH", "LD_*"), + ("ld_audit", "LD_*"), + ("BASH_ENV", "BASH_ENV"), + ("ENV", "ENV"), + ("BASH_FUNC_injected%%", "BASH_FUNC_*"), + ("PYTHONHOME", "PYTHON*"), + ("PYTHONPATH", "PYTHON*"), + ("PYTHONHTTPSVERIFY", "PYTHON*"), + ("pythonstartup", "PYTHON*"), + ("HTTP_PROXY", "HTTP_PROXY"), + ("https_proxy", "HTTPS_PROXY"), + ("ALL_PROXY", "ALL_PROXY"), + ("no_proxy", "NO_PROXY"), + ("SSL_CERT_FILE", "SSL_CERT_FILE"), + ("ssl_cert_dir", "SSL_CERT_DIR"), + ("REQUESTS_CA_BUNDLE", "REQUESTS_CA_BUNDLE"), + ("curl_ca_bundle", "CURL_CA_BUNDLE"), + ("SSLKEYLOGFILE", "SSLKEYLOGFILE"), + ("TELEO_CLOUDSQL_CREDENTIAL_MODE", "TELEO_CLOUDSQL_CREDENTIAL_MODE"), + ("TELEO_KB_CLAIM_BASE_URL", "TELEO_KB_CLAIM_BASE_URL"), + ("TELEO_KB_READ_ATTEMPTS", "TELEO_KB_READ_ATTEMPTS"), + ("TELEO_MEMORY_INCLUDE_EXCERPTS", "TELEO_MEMORY_INCLUDE_EXCERPTS"), + ("TELEO_MEMORY_REDACT", "TELEO_MEMORY_REDACT"), + ], +) +def test_environment_file_broader_credentials_and_forbidden_fields_fail_closed( + field: str, + safe_label: str, +) -> None: + environment = verifier.expected_environment() + injected_value = "sensitive-injected-value" + environment[field] = injected_value + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert caught.value.code == "forbidden_environment_fields" + assert caught.value.fields == (safe_label,) + assert injected_value not in rendered + + +def test_arbitrary_forbidden_key_suffix_is_not_an_output_channel() -> None: + environment = verifier.expected_environment() + environment["PG_SECRET_MARKER"] = "not-output" + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert "PG_SECRET_MARKER" not in rendered + assert "PG*" in rendered + + +def test_pythonunbuffered_is_the_only_allowed_python_runtime_tuning() -> None: + environment = verifier.expected_environment() + environment["PYTHONUNBUFFERED"] = "1" + + assert verifier.validate_environment(environment) == tuple(sorted(verifier.expected_environment())) + + +def test_bash_env_executes_before_a_script_but_effective_verifier_rejects_it(tmp_path: Path) -> None: + marker = "BASH_ENV_EXECUTED" + payload = tmp_path / "bash-env.sh" + payload.write_text(f"printf '{marker}\\n'\n", encoding="utf-8") + completed = subprocess.run( + ["/bin/bash", "-c", ":"], + env={"BASH_ENV": str(payload), "PATH": verifier.RUNTIME_PATH}, + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 0 + assert completed.stdout.strip() == marker + + environment = verifier.expected_environment() + environment["BASH_ENV"] = str(payload) + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + assert caught.value.fields == ("BASH_ENV",) + + +def test_admin_secret_reference_in_any_value_fails_without_field_or_value_leak() -> None: + environment = verifier.expected_environment() + environment["UNRELATED"] = f"prefix/{verifier.ADMINISTRATOR_PASSWORD_SECRET}/suffix" + + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_environment(environment) + + rendered = verifier.canonical_json(verifier.failure_receipt(caught.value)) + assert caught.value.code == "administrator_secret_reference" + assert verifier.ADMINISTRATOR_PASSWORD_SECRET not in rendered + assert "UNRELATED" not in rendered + + +@pytest.mark.parametrize( + "raw", + [ + b"A=1", + b"A=1\0\0", + b"A=1\0BROKEN\0", + b"=value\0", + b"A=1\0A=2\0", + b"A=\xff\0", + b"", + ], +) +def test_malformed_nul_environment_is_rejected(raw: bytes) -> None: + with pytest.raises(verifier.VerificationError) as caught: + verifier.parse_environment(raw) + + assert caught.value.code == "environment_malformed" + + +def test_parser_preserves_equals_inside_values() -> None: + assert verifier.parse_environment(b"A=one=two\0B=\0") == {"A": "one=two", "B": ""} + + +@pytest.mark.parametrize("value", [0, -1, 1.5, "0", "01", "+1", "-3", "not-a-pid", True]) +def test_pid_must_be_a_positive_integer(value: object) -> None: + with pytest.raises(verifier.VerificationError) as caught: + verifier.validate_pid(value) # type: ignore[arg-type] + + assert caught.value.code == "invalid_pid" + + +def test_cli_errors_are_canonical_sanitized_json_and_exit_one(capsys: pytest.CaptureFixture[str]) -> None: + secret_input = "not-a-pid-secret-marker" + + returncode = verifier.main(["--pid", secret_input]) + + output = capsys.readouterr() + assert returncode == 1 + assert output.err == "" + assert json.loads(output.out) == {"error": {"code": "invalid_pid"}, "status": "fail"} + assert secret_input not in output.out + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="requires Linux /proc/PID/environ") +def test_cli_reads_real_process_environment_and_detects_effective_conflict( + capsys: pytest.CaptureFixture[str], +) -> None: + expected = verifier.expected_environment() + good_process = subprocess.Popen( + ["/bin/sleep", "30"], + env=expected, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + wait_for_process_environment(good_process, expected) + assert verifier.main(["--pid", str(good_process.pid)]) == 0 + good_output = capsys.readouterr() + assert good_output.err == "" + assert json.loads(good_output.out)["status"] == "pass" + finally: + good_process.terminate() + good_process.wait(timeout=5) + + conflicting = dict(expected) + conflicting["TELEO_CLOUDSQL_USER"] = "postgres" + conflicting["GOOGLE_APPLICATION_CREDENTIALS"] = "/tmp/broader-credential.json" + bad_process = subprocess.Popen( + ["/bin/sleep", "30"], + env=conflicting, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + wait_for_process_environment(bad_process, conflicting) + assert verifier.main(["--pid", str(bad_process.pid)]) == 1 + bad_output = capsys.readouterr() + receipt = json.loads(bad_output.out) + assert bad_output.err == "" + assert receipt == { + "error": {"code": "forbidden_environment_fields", "fields": ["GOOGLE_APPLICATION_CREDENTIALS"]}, + "status": "fail", + } + assert "postgres" not in bad_output.out + assert "/tmp/broader-credential.json" not in bad_output.out + finally: + bad_process.terminate() + bad_process.wait(timeout=5) + + +def test_unexpected_read_error_is_generic_and_does_not_echo_path( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + sensitive_marker = "sensitive-proc-read-detail" + + def fail_read(_self: object) -> bytes: + raise OSError(sensitive_marker) + + monkeypatch.setattr(verifier.Path, "read_bytes", fail_read) + + assert verifier.main(["--pid", str(os.getpid())]) == 1 + output = capsys.readouterr() + assert json.loads(output.out) == {"error": {"code": "internal_error"}, "status": "fail"} + assert sensitive_marker not in output.out diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index 4c6daaa..3f9a370 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -3,12 +3,14 @@ from __future__ import annotations import ast +import base64 import contextlib import importlib.util import io import json import os import re +import shutil import subprocess from pathlib import Path from types import SimpleNamespace @@ -23,6 +25,9 @@ from scripts.working_leo_open_ended_benchmark import ( ROOT = Path(__file__).resolve().parents[1] BRIDGE_DIR = ROOT / "hermes-agent" / "leoclean-bin" DB_CONTEXT_PLUGIN = ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py" +GCP_RUNTIME_ROLE_SQL = ROOT / "ops" / "gcp_leoclean_runtime_role.sql" +GCP_RUNTIME_WRAPPER = BRIDGE_DIR / "teleo-kb-gcp" +GCP_RUNTIME_PROVISIONER = ROOT / "ops" / "provision_gcp_leoclean_runtime_role.sh" def test_leoclean_bridge_files_are_present_and_parseable() -> None: @@ -38,6 +43,71 @@ def test_leoclean_bridge_files_are_present_and_parseable() -> None: subprocess.run(["bash", "-n", str(wrapper)], check=True) +@pytest.mark.parametrize( + "argument", + [ + "--host", + "--host=127.0.0.1", + "--port", + "--port=6543", + "--db", + "--db=postgres", + "--canonical-db", + "--canonical-db=postgres", + "--user", + "--user=postgres", + "--password-secret", + "--password-secret=administrator-secret", + "--project", + "--project=other-project", + "--credential-mode", + "--credential-mode=clone-readonly", + ], +) +def test_deployed_gcp_wrapper_executable_rejects_every_identity_override(argument: str) -> None: + injected = "sensitive-override-value" + completed = subprocess.run( + [str(GCP_RUNTIME_WRAPPER), argument, injected], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 64 + assert "identity and endpoint overrides are forbidden" in completed.stderr + assert injected not in completed.stderr + assert completed.stdout == "" + + +def test_gcp_wrapper_and_password_provisioner_are_executable_and_syntax_valid() -> None: + assert os.access(GCP_RUNTIME_WRAPPER, os.X_OK) + assert os.access(GCP_RUNTIME_PROVISIONER, os.X_OK) + subprocess.run(["bash", "-n", str(GCP_RUNTIME_WRAPPER)], check=True) + subprocess.run(["bash", "-n", str(GCP_RUNTIME_PROVISIONER)], check=True) + + wrapper = GCP_RUNTIME_WRAPPER.read_text(encoding="utf-8") + assert "exec /usr/bin/env -i" in wrapper + assert '/usr/bin/python3 -I "$CLOUDSQL_TOOL" "$@"' in wrapper + assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in wrapper + assert "PGSSLMODE=verify-ca" in wrapper + assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in wrapper + + provisioner = GCP_RUNTIME_PROVISIONER.read_text(encoding="utf-8") + capture_runtime = "runtime_password=${TELEO_LEOCLEAN_DB_PASSWORD-}" + capture_admin = "administrator_password=${PGPASSWORD-}" + clear_ambient = "unset TELEO_LEOCLEAN_DB_PASSWORD PGPASSWORD" + first_external_child = 'SCRIPT_DIR="$(cd -- "$script_directory" && pwd -P)"' + assert provisioner.index(capture_runtime) < provisioner.index(first_external_child) + assert provisioner.index(capture_admin) < provisioner.index(first_external_child) + assert provisioner.index(clear_ambient) < provisioner.index(first_external_child) + assert '"$SETSID_BIN" -- "$PSQL_BIN"' in provisioner + assert 'kill -TERM -- "-$provision_pid"' in provisioner + assert 'wait "$provision_pid"' in provisioner + assert "SERVER_CA_SHA256=80701e" in provisioner + assert "--no-password" in provisioner + assert "PGSSLMODE=verify-ca" in provisioner + + def test_vps_bridge_recognizes_natural_mixed_briefing_contract() -> None: module = _load_module(BRIDGE_DIR / "kb_tool.py") contracts = module.operational_contracts( @@ -98,12 +168,20 @@ def test_cloudsql_wrapper_supports_expected_review_gated_commands() -> None: assert command in cloudsql_text assert "status" in vps_text + assert "SCRIPT_DIR=${SCRIPT_PATH%/*}" in wrapper_text + assert 'SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"' in wrapper_text + assert 'CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py"' in wrapper_text + assert wrapper_text.startswith("#!/bin/bash\n") + assert "exec /usr/bin/python3" in wrapper_text + assert "command -v psql" in wrapper_text + assert "gcloud" not in wrapper_text + assert '"gcloud"' not in cloudsql_text assert "kb_stage.kb_proposals" in cloudsql_text assert "canonical apply" in cloudsql_text.lower() -def _install_wrapper_fixture(tmp_path: Path) -> tuple[Path, dict[str, str]]: +def _install_wrapper_fixture(tmp_path: Path, *, missing: frozenset[str] = frozenset()) -> tuple[Path, dict[str, str]]: profile = tmp_path / "profile" bin_dir = profile / "bin" bin_dir.mkdir(parents=True) @@ -111,11 +189,12 @@ def _install_wrapper_fixture(tmp_path: Path) -> tuple[Path, dict[str, str]]: wrapper.write_bytes((BRIDGE_DIR / "teleo-kb").read_bytes()) wrapper.chmod(0o700) cloudsql = bin_dir / "cloudsql_memory_tool.py" - cloudsql.write_text( - "import json, sys\nprint(json.dumps({'backend': 'cloudsql', 'argv': sys.argv[1:]}))\n", - encoding="utf-8", - ) - cloudsql.chmod(0o700) + if "cloudsql-tool" not in missing: + cloudsql.write_text( + "import json, sys\nprint(json.dumps({'backend': 'cloudsql', 'argv': sys.argv[1:]}))\n", + encoding="utf-8", + ) + cloudsql.chmod(0o700) local = bin_dir / "kb_tool.py" local.write_text( "import json, sys\nprint(json.dumps({'backend': 'local', 'argv': sys.argv[1:]}))\n", @@ -124,7 +203,13 @@ def _install_wrapper_fixture(tmp_path: Path) -> tuple[Path, dict[str, str]]: local.chmod(0o700) fake_bin = tmp_path / "fake-bin" fake_bin.mkdir() - for name in ("psql", "gcloud"): + for name in ("bash", "python3"): + target = shutil.which(name) + assert target + (fake_bin / name).symlink_to(target) + for name in ("psql",): + if name in missing: + continue executable = fake_bin / name executable.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") executable.chmod(0o700) @@ -132,7 +217,7 @@ def _install_wrapper_fixture(tmp_path: Path) -> tuple[Path, dict[str, str]]: **os.environ, "HERMES_HOME": str(profile), "TELEO_KB_MODE": "auto", - "PATH": f"{fake_bin}:{os.environ['PATH']}", + "PATH": str(fake_bin), } return wrapper, env @@ -173,6 +258,40 @@ def test_cloudsql_wrapper_auto_mode_never_falls_back_after_supported_command_fai assert "local" not in completed.stdout +@pytest.mark.parametrize("missing", ["cloudsql-tool", "psql"]) +def test_cloudsql_wrapper_auto_mode_fails_closed_when_prerequisite_is_missing(tmp_path: Path, missing: str) -> None: + wrapper, env = _install_wrapper_fixture(tmp_path, missing=frozenset({missing})) + + completed = subprocess.run( + [str(wrapper), "search", "canonical only", "--format", "json"], + text=True, + capture_output=True, + check=False, + env=env, + ) + + assert completed.returncode == 69 + assert "local" not in completed.stdout + assert "Cloud SQL" in completed.stderr + + +def test_cloudsql_wrapper_explicit_mode_rejects_unsupported_commands_without_fallback(tmp_path: Path) -> None: + wrapper, env = _install_wrapper_fixture(tmp_path) + env["TELEO_KB_MODE"] = "cloudsql" + + completed = subprocess.run( + [str(wrapper), "prepare-source", "example.md"], + text=True, + capture_output=True, + check=False, + env=env, + ) + + assert completed.returncode == 64 + assert "local" not in completed.stdout + assert "not supported by the Cloud SQL backend" in completed.stderr + + def test_bridge_edge_proposals_stage_strict_apply_payloads() -> None: cloudsql_text = (BRIDGE_DIR / "cloudsql_memory_tool.py").read_text() vps_text = (BRIDGE_DIR / "kb_tool.py").read_text() @@ -184,6 +303,9 @@ def test_bridge_edge_proposals_stage_strict_apply_payloads() -> None: assert "'to_claim', to_row.id::text" in text assert "'edge_type', p.edge_type::text" in text + assert "kb_stage.stage_leoclean_proposal(" in cloudsql_text + assert "insert into kb_stage.kb_proposals" not in cloudsql_text.lower() + def test_bridge_source_does_not_commit_raw_secret_values() -> None: combined = "\n".join(path.read_text(errors="replace") for path in BRIDGE_DIR.iterdir() if path.is_file()) @@ -199,11 +321,704 @@ def test_bridge_source_does_not_commit_raw_secret_values() -> None: assert not re.search(pattern, combined), pattern +def _runtime_connection_args(module, **overrides): + values = { + "credential_mode": "runtime", + "user": module.RUNTIME_DB_USER, + "password_secret": module.RUNTIME_PASSWORD_SECRET, + "project": module.RUNTIME_GCP_PROJECT, + "host": module.RUNTIME_CLOUDSQL_HOST, + "port": module.RUNTIME_CLOUDSQL_PORT, + "db": module.RUNTIME_CLOUDSQL_DB, + "canonical_db": module.RUNTIME_CLOUDSQL_DB, + "command": "status", + } + values.update(overrides) + return SimpleNamespace(**values) + + +def _configure_runtime_environment(monkeypatch: pytest.MonkeyPatch) -> None: + for key in tuple(os.environ): + normalized_key = key.upper() + if ( + normalized_key.startswith("PG") + or normalized_key.startswith("CLOUDSDK_") + or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS" + or normalized_key.startswith("LD_") + or normalized_key.startswith("BASH_FUNC_") + or normalized_key.startswith("PYTHON") + or normalized_key + in { + "BASH_ENV", + "BASHOPTS", + "CDPATH", + "ENV", + "GLOBIGNORE", + "IFS", + "PS4", + "SHELLOPTS", + } + or normalized_key == "PROXY" + or normalized_key.endswith("_PROXY") + or normalized_key + in { + "CURL_CA_BUNDLE", + "PYTHONHTTPSVERIFY", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SSLKEYLOGFILE", + "OPENSSL_CONF", + "OPENSSL_ENGINES", + "OPENSSL_MODULES", + "GCONV_PATH", + } + ): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("TELEO_GCP_METADATA_ONLY", "1") + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") + + +def test_runtime_environment_fixture_scrubs_github_runner_inheritance( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + monkeypatch.setenv("CI", "true") + monkeypatch.setenv("LD_LIBRARY_PATH", "/opt/hostedtoolcache/Python/3.11.15/x64/lib") + monkeypatch.setenv("pythonLocation", "/opt/hostedtoolcache/Python/3.11.15/x64") + monkeypatch.setenv("HTTP_PROXY", "http://runner-proxy.invalid:8080") + monkeypatch.setenv("BASH_FUNC_runner%%", "() { :; }") + monkeypatch.setenv("SSL_CERT_FILE", "/tmp/runner-ca.pem") + monkeypatch.setenv("OPENSSL_CONF", "/tmp/runner-openssl.cnf") + monkeypatch.setenv("GCONV_PATH", "/tmp/runner-gconv") + + _configure_runtime_environment(monkeypatch) + + assert os.environ["CI"] == "true" + module.validate_runtime_connection(args) + + +class _FakeHttpResponse: + def __init__(self, body: bytes, status: int = 200) -> None: + self.body = body + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + def getcode(self) -> int: + return self.status + + def read(self, _limit: int) -> bytes: + return self.body + + +def _runtime_http_bodies(module, password: bytes = b"scoped-runtime-password") -> list[bytes]: + return [ + module.RUNTIME_SERVICE_ACCOUNT.encode("ascii"), + json.dumps({"access_token": "metadata-access-token", "token_type": "Bearer", "expires_in": 300}).encode(), + json.dumps({"payload": {"data": base64.b64encode(password).decode("ascii")}}).encode(), + ] + + +def test_cloudsql_runtime_requires_the_exact_scoped_identity(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + monkeypatch.delenv("PGPASSWORD", raising=False) + + with pytest.raises(SystemExit, match="requires database user leoclean_kb_runtime"): + module.validate_runtime_connection(SimpleNamespace(credential_mode="runtime", user=None, password_secret=None)) + with pytest.raises(SystemExit, match="requires database user leoclean_kb_runtime"): + module.validate_runtime_connection( + SimpleNamespace( + credential_mode="runtime", + user="postgres", + password_secret=module.RUNTIME_PASSWORD_SECRET, + ) + ) + with pytest.raises(SystemExit, match="requires database user leoclean_kb_runtime"): + module.validate_runtime_connection( + SimpleNamespace( + credential_mode="runtime", + user=f" {module.RUNTIME_DB_USER}", + password_secret=module.RUNTIME_PASSWORD_SECRET, + ) + ) + with pytest.raises(SystemExit, match="requires scoped password secret"): + module.validate_runtime_connection( + SimpleNamespace( + credential_mode="runtime", + user=module.RUNTIME_DB_USER, + password_secret="gcp-teleo-pgvector-standby-postgres-password", + ) + ) + with pytest.raises(SystemExit, match="requires scoped password secret"): + module.validate_runtime_connection( + SimpleNamespace(credential_mode="runtime", user=module.RUNTIME_DB_USER, password_secret=None) + ) + + +def test_cloudsql_runtime_accepts_only_scoped_credentials_and_rejects_inherited_password( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + + module.validate_runtime_connection(args) + + monkeypatch.setenv("PGPASSWORD", "") + with pytest.raises(SystemExit, match="Refusing inherited credential, proxy, TLS, or PostgreSQL environment"): + module.validate_runtime_connection(args) + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("GOOGLE_APPLICATION_CREDENTIALS", "/tmp/ambient.json"), + ("google_application_credentials", "/tmp/ambient-lowercase.json"), + ("CLOUDSDK_CONFIG", "/tmp/gcloud"), + ("CLOUDSDK_AUTH_ACCESS_TOKEN", "ambient-token"), + ("cloudsdk_core_project", "attacker-project"), + ("cloudsdk_auth_access_token", "ambient-lowercase-token"), + ("PGHOST", "attacker.invalid"), + ("PGOPTIONS", "-c search_path=attacker"), + ("pgservice", "attacker-service"), + ("ld_preload", "/tmp/attacker.so"), + ("LD_LIBRARY_PATH", "/tmp/attacker-library"), + ("Ld_AuDiT", "/tmp/attacker-audit.so"), + ("BASH_ENV", "/tmp/attacker-bash-env"), + ("ENV", "/tmp/attacker-sh-env"), + ("BASH_FUNC_psql%%", "() { :; }"), + ("PythonHome", "/tmp/attacker-python"), + ("pythonpath", "/tmp/attacker-modules"), + ("PYTHONHTTPSVERIFY", "0"), + ("pythonstartup", "/tmp/attacker-startup.py"), + ("https_proxy", "http://attacker.invalid:8080"), + ("HtTp_PrOxY", "http://attacker.invalid:8080"), + ("ssl_cert_file", "/tmp/attacker.pem"), + ("SSL_CERT_DIR", "/tmp/attacker-certs"), + ("REQUESTS_CA_BUNDLE", "/tmp/attacker.pem"), + ("SSLKEYLOGFILE", "/tmp/attacker-tls-keys"), + ], +) +def test_cloudsql_runtime_rejects_ambient_auth_and_pg_environment( + monkeypatch: pytest.MonkeyPatch, + name: str, + value: str, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.setenv(name, value) + + with pytest.raises(SystemExit, match="Refusing inherited credential, proxy, TLS, or PostgreSQL environment"): + module.validate_runtime_connection(args) + + +def test_cloudsql_runtime_allows_only_pythonunbuffered_runtime_tuning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.setenv("PYTHONUNBUFFERED", "1") + + module.validate_runtime_connection(args) + + +def test_cloudsql_runtime_requires_explicit_metadata_only_mode(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.delenv("TELEO_GCP_METADATA_ONLY") + + with pytest.raises(SystemExit, match="TELEO_GCP_METADATA_ONLY=1"): + module.validate_runtime_connection(args) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("project", "other-project", "project=teleo-501523"), + ("host", "127.0.0.1", "host=10.61.0.3"), + ("port", "6543", "port=5432"), + ("db", "teleo_kb", "db=teleo_canonical"), + ("canonical_db", "teleo_kb", "canonical_db=teleo_canonical"), + ], +) +def test_cloudsql_runtime_rejects_any_target_drift_before_credentials( + monkeypatch: pytest.MonkeyPatch, + field: str, + value: str, + message: str, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module, **{field: value}) + monkeypatch.delenv("PGPASSWORD", raising=False) + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") + + with pytest.raises(SystemExit, match=message): + module.validate_runtime_connection(args) + + +def test_cloudsql_runtime_rejects_audit_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + monkeypatch.delenv("PGPASSWORD", raising=False) + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "1") + + with pytest.raises(SystemExit, match="AUDIT_FALLBACK=0"): + module.validate_runtime_connection(args) + + +def test_cloudsql_clone_mode_is_read_only_and_bound_to_one_disposable_database( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = SimpleNamespace( + credential_mode="clone-readonly", + user="postgres", + password_secret=None, + canonical_db="teleo_clone_canary_123", + db="teleo_clone_canary_123", + command="status", + ) + monkeypatch.setenv("PGPASSWORD", "clone-only-password") + monkeypatch.setenv("PGOPTIONS", module.CLONE_READ_ONLY_PGOPTIONS) + + module.validate_runtime_connection(args) + assert module.password(args) == "clone-only-password" + + args.command = "propose-core-change" + with pytest.raises(SystemExit, match="read commands only"): + module.validate_runtime_connection(args) + args.command = "status" + args.canonical_db = "teleo_canonical" + with pytest.raises(SystemExit, match=r"teleo_clone_\*"): + module.validate_runtime_connection(args) + + +def test_cloudsql_clone_mode_never_resolves_a_password_secret_argument(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = SimpleNamespace( + credential_mode="clone-readonly", + user="postgres", + password_secret="gcp-teleo-pgvector-standby-postgres-password", + canonical_db="teleo_clone_canary_123", + db="teleo_clone_canary_123", + command="status", + ) + monkeypatch.delenv("PGPASSWORD", raising=False) + monkeypatch.setenv("PGOPTIONS", module.CLONE_READ_ONLY_PGOPTIONS) + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_args, **_kwargs: pytest.fail("a password-secret argument must never launch a credential helper"), + ) + + with pytest.raises(SystemExit, match="requires an explicit user and inherited test credential"): + module.password(args) + + +def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + monkeypatch.setenv("HOME", "/tmp/untrusted-home") + monkeypatch.setenv("LANG", "attacker-locale") + monkeypatch.setenv("SHELL", "/tmp/untrusted-shell") + monkeypatch.setenv("UNRELATED_AMBIENT_VALUE", "must-not-reach-psql") + captured: dict[str, object] = {} + + def fake_run(command, **kwargs): + captured["command"] = command + captured["kwargs"] = kwargs + return SimpleNamespace(returncode=0, stdout="ok\n", stderr="") + + monkeypatch.setattr(module, "runtime_password_from_metadata", lambda: "scoped-runtime-password") + monkeypatch.setattr(module.subprocess, "run", fake_run) + + assert module.run_psql(args, "select 1;") == "ok\n" + assert captured["command"] == ["/usr/bin/psql", "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"] + kwargs = captured["kwargs"] + assert isinstance(kwargs, dict) + env = kwargs["env"] + assert env == { + "PATH": "/usr/bin:/bin", + "PGHOST": "10.61.0.3", + "PGPORT": "5432", + "PGDATABASE": "teleo_canonical", + "PGUSER": module.RUNTIME_DB_USER, + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": module.RUNTIME_SSL_ROOT_CERT, + "PGCONNECT_TIMEOUT": "8", + "PGPASSWORD": "scoped-runtime-password", + } + + +def test_cloudsql_runtime_metadata_and_secret_requests_are_exact_and_silent( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + bodies = _runtime_http_bodies(module) + requests: list[tuple[str, str, dict[str, str], float]] = [] + + def fake_urlopen(request, *, timeout): + requests.append( + ( + request.full_url, + request.get_method(), + {key.lower(): value for key, value in request.header_items()}, + timeout, + ) + ) + return _FakeHttpResponse(bodies.pop(0)) + + monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", fake_urlopen) + + assert module.password(args) == "scoped-runtime-password" + assert module.GCE_METADATA_EMAIL_URL == ( + "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" + ) + assert module.GCE_METADATA_TOKEN_URL == ( + "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" + ) + assert requests == [ + ( + module.GCE_METADATA_EMAIL_URL, + "GET", + {"metadata-flavor": "Google"}, + module.HTTP_TIMEOUT_SECONDS, + ), + ( + module.GCE_METADATA_TOKEN_URL, + "GET", + {"metadata-flavor": "Google"}, + module.HTTP_TIMEOUT_SECONDS, + ), + ( + module.RUNTIME_SECRET_ACCESS_URL, + "GET", + {"accept": "application/json", "authorization": "Bearer metadata-access-token"}, + module.HTTP_TIMEOUT_SECONDS, + ), + ] + assert capsys.readouterr() == ("", "") + + +def test_cloudsql_runtime_http_opener_disables_proxies_and_redirects() -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + proxy_handlers = [ + handler + for handler in module.RUNTIME_HTTP_OPENER.handlers + if isinstance(handler, module.urllib.request.ProxyHandler) + ] + redirect_handlers = [ + handler + for handler in module.RUNTIME_HTTP_OPENER.handlers + if isinstance(handler, module.urllib.request.HTTPRedirectHandler) + ] + + # An empty ProxyHandler suppresses build_opener's environment-backed default; + # urllib omits the empty handler itself from the final chain. + assert proxy_handlers == [] + assert module.RUNTIME_PROXY_HANDLER.proxies == {} + assert len(redirect_handlers) == 1 + assert type(redirect_handlers[0]) is module._RejectRedirectHandler + + +def test_cloudsql_runtime_rejects_wrong_metadata_service_account_before_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + calls: list[str] = [] + + def fake_urlopen(request, *, timeout): + assert timeout == module.HTTP_TIMEOUT_SECONDS + calls.append(request.full_url) + return _FakeHttpResponse(b"unexpected-sa@teleo-501523.iam.gserviceaccount.com") + + monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", fake_urlopen) + + with pytest.raises(SystemExit, match="did not match the required service account"): + module.password(args) + assert calls == [module.GCE_METADATA_EMAIL_URL] + + +@pytest.mark.parametrize( + "token_body", + [ + b"not-json", + b"[]", + b'{"access_token":"token","token_type":"Basic","expires_in":300}', + b'{"access_token":"token with spaces","token_type":"Bearer","expires_in":300}', + b'{"access_token":"token","token_type":"Bearer","expires_in":0}', + b'{"access_token":"one","access_token":"two","token_type":"Bearer","expires_in":300}', + ], +) +def test_cloudsql_runtime_rejects_malformed_metadata_tokens_without_leaking_them( + monkeypatch: pytest.MonkeyPatch, + token_body: bytes, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + bodies = [module.RUNTIME_SERVICE_ACCOUNT.encode("ascii"), token_body] + monkeypatch.setattr( + module.RUNTIME_HTTP_OPENER, + "open", + lambda _request, *, timeout: _FakeHttpResponse(bodies.pop(0)), + ) + + with pytest.raises(SystemExit, match="Runtime metadata token response was invalid") as captured: + module.password(args) + assert "token with spaces" not in str(captured.value) + + +@pytest.mark.parametrize( + "secret_body", + [ + b"not-json", + b"[]", + b"{}", + b'{"payload":{"data":"%%%"}}', + json.dumps({"payload": {"data": base64.b64encode(b"x" * 4097).decode("ascii")}}).encode(), + json.dumps({"payload": {"data": base64.b64encode(b"line1\nline2").decode("ascii")}}).encode(), + json.dumps({"payload": {"data": base64.b64encode(b"line1\rline2").decode("ascii")}}).encode(), + json.dumps({"payload": {"data": base64.b64encode(b"nul\x00byte").decode("ascii")}}).encode(), + ], +) +def test_cloudsql_runtime_rejects_malformed_secret_payloads_without_leaking_them( + monkeypatch: pytest.MonkeyPatch, + secret_body: bytes, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + bodies = _runtime_http_bodies(module) + bodies[-1] = secret_body + monkeypatch.setattr( + module.RUNTIME_HTTP_OPENER, + "open", + lambda _request, *, timeout: _FakeHttpResponse(bodies.pop(0)), + ) + + with pytest.raises(SystemExit, match=r"Scoped runtime secret .* invalid") as captured: + module.password(args) + assert secret_body.decode("utf-8", errors="ignore") not in str(captured.value) + + +def test_cloudsql_runtime_admin_secret_is_impossible_through_arguments( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module, password_secret="gcp-teleo-pgvector-standby-postgres-password") + _configure_runtime_environment(monkeypatch) + monkeypatch.setattr( + module.RUNTIME_HTTP_OPENER, + "open", + lambda *_args, **_kwargs: pytest.fail("metadata must not be queried for an admin secret argument"), + ) + + with pytest.raises(SystemExit, match="requires scoped password secret"): + module.password(args) + + +def test_cloudsql_runtime_http_and_psql_failures_never_echo_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + _configure_runtime_environment(monkeypatch) + dummy = "DUMMY-RUNTIME-SECRET-DO-NOT-LEAK" + + def failed_urlopen(_request, *, timeout): + raise RuntimeError(f"provider echoed {dummy}") + + monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", failed_urlopen) + with pytest.raises(SystemExit) as http_error: + module.password(args) + assert dummy not in str(http_error.value) + + monkeypatch.setattr(module, "runtime_password_from_metadata", lambda: dummy) + monkeypatch.setattr( + module.subprocess, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=2, stdout=dummy, stderr=dummy), + ) + with pytest.raises(SystemExit, match="Cloud SQL query failed with exit status 2") as psql_error: + module.run_psql(args, "select 1;") + assert dummy not in str(psql_error.value) + + +def test_cloudsql_runtime_read_marker_binds_database_user_private_server_and_ssl( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + marker = { + "database": module.RUNTIME_CLOUDSQL_DB, + "database_user": module.RUNTIME_DB_USER, + "server_addr": module.RUNTIME_CLOUDSQL_HOST, + "server_port": int(module.RUNTIME_CLOUDSQL_PORT), + "ssl": True, + "ssl_version": "TLSv1.3", + "system_identifier": module.RUNTIME_SYSTEM_IDENTIFIER, + "wal_lsn": "0/1", + } + captured: dict[str, object] = {} + + def fake_psql_json_lines(_args, sql, db=None): + captured["sql"] = sql + captured["db"] = db + return [marker] + + monkeypatch.setattr(module, "psql_json_lines", fake_psql_json_lines) + + assert module.database_read_marker(args) == marker + assert captured["db"] == module.RUNTIME_CLOUDSQL_DB + assert "inet_server_addr()" in str(captured["sql"]) + assert "pg_stat_ssl" in str(captured["sql"]) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("database", "teleo_kb", "unexpected database identity"), + ("database_user", "postgres", "unexpected database identity"), + ("server_addr", "127.0.0.1", "unexpected server address"), + ("server_port", 6543, "unexpected server port"), + ("ssl", False, "encrypted connection"), + ], +) +def test_cloudsql_runtime_read_marker_rejects_wrong_live_identity( + monkeypatch: pytest.MonkeyPatch, + field: str, + value: object, + message: str, +) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + args = _runtime_connection_args(module) + marker = { + "database": module.RUNTIME_CLOUDSQL_DB, + "database_user": module.RUNTIME_DB_USER, + "server_addr": module.RUNTIME_CLOUDSQL_HOST, + "server_port": int(module.RUNTIME_CLOUDSQL_PORT), + "ssl": True, + "ssl_version": "TLSv1.3", + "system_identifier": module.RUNTIME_SYSTEM_IDENTIFIER, + "wal_lsn": "0/1", + } + marker[field] = value + monkeypatch.setattr(module, "psql_json_lines", lambda *_args, **_kwargs: [marker]) + + with pytest.raises(SystemExit, match=message): + module.database_read_marker(args) + + +def test_cloudsql_proposal_calls_leave_leo_identity_to_the_database(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + captured: list[str] = [] + + def fake_psql_json_lines(_args, sql, db=None): + captured.append(sql) + return [{"id": "proposal-id", "database": db}] + + monkeypatch.setattr(module, "psql_json_lines", fake_psql_json_lines) + core_args = SimpleNamespace( + proposal_type="revise_claim", + target_kind="claim", + target_ref="claim-id", + current="old", + proposed="new", + evidence=[], + implication=[], + originator="", + channel="telegram", + source_ref="telegram:1", + rationale="Grounded correction", + canonical_db="teleo_canonical", + ) + edge_args = SimpleNamespace( + from_claim="11111111-1111-4111-8111-111111111111", + to_claim="22222222-2222-4222-8222-222222222222", + edge_type="supports", + channel="telegram", + source_ref="telegram:2", + rationale="Grounded relation", + canonical_db="teleo_canonical", + ) + + module.propose_core_change(core_args) + module.propose_edge(edge_args) + + assert "p.proposed_by_handle" not in captured[0] + assert "as proposed_by_handle" not in captured[0] + assert "p.proposed_by_handle" not in captured[1] + assert "as proposed_by_handle" not in captured[1] + assert "stage_leoclean_proposal(\n p.proposal_type,\n p.channel," in captured[0] + assert "stage_leoclean_proposal(\n 'add_edge',\n p.channel," in captured[1] + + +def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None: + sql = GCP_RUNTIME_ROLE_SQL.read_text() + + assert "security definer" in sql.lower() + assert "stage_leoclean_proposal" in sql + assert "'pending_review'" in sql + assert "revoke all privileges on all tables in schema public, kb_stage" in sql + assert "grant insert" in sql.lower() + assert "unexpectedly has database CREATE" in sql + assert "unexpectedly has schema CREATE" in sql + assert "runtime CONNECT reaches a noncanonical database" in sql + assert "PUBLIC retains database CONNECT" in sql + assert "PUBLIC retains large-object mutator EXECUTE" in sql + assert "REVOKE TEMP FROM PUBLIC would be a" in sql + assert "can execute unexpected SECURITY DEFINER functions" in sql + assert "revoke connect on database %I from public" in sql + assert "revoke execute on function pg_catalog.%I(%s) from public" in sql + assert re.search( + r"grant\s+insert\s*\([^)]*proposal_type[^)]*payload[^)]*\)\s+" + r"on\s+kb_stage\.kb_proposals\s+to\s+leoclean_kb_stage_owner", + sql, + re.IGNORECASE | re.DOTALL, + ) + assert not re.search( + r"grant\s+insert(?:\s*\([^)]*\))?\s+on\s+[^;]+\s+to\s+leoclean_kb_runtime", + sql, + re.IGNORECASE | re.DOTALL, + ) + assert "grant update" not in sql.lower() + assert "grant delete" not in sql.lower() + assert "create role leoclean_kb_runtime nologin" in sql.lower() + disable_statement = "alter role leoclean_kb_runtime\n with nologin nosuperuser" + password_statement = "\\password leoclean_kb_runtime" + assert sql.lower().index(disable_statement) < sql.index(password_statement) + assert "\\getenv runtime_password" not in sql + assert "password :'runtime_password'" not in sql.lower() + assert sql.lower().index("with nologin nosuperuser") < sql.index("\\connect teleo_canonical") + assert sql.index("\\connect teleo_canonical") < sql.lower().rindex("with login nosuperuser") + assert sql.lower().rindex("with login nosuperuser") < sql.rindex("commit;") + assert "leoclean_kb_runtime final login attributes are unsafe" in sql + + def _load_module(path: Path): spec = importlib.util.spec_from_file_location(path.stem, path) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + if path.name == "cloudsql_memory_tool.py": + module.runtime_ssl_root_cert = lambda: module.RUNTIME_SSL_ROOT_CERT + module.runtime_cloudsdk_config = lambda: module.RUNTIME_CLOUDSDK_CONFIG return module @@ -359,11 +1174,16 @@ def test_cloudsql_status_works_without_optional_audit_fallback(monkeypatch) -> N assert all("teleo_restore.response_audit" not in sql or "to_regclass" in sql for sql, _db in calls) -def test_cloudsql_zero_hit_search_stays_canonical_when_audit_is_unavailable(monkeypatch) -> None: +def test_cloudsql_zero_hit_search_stays_canonical_when_audit_is_disabled(monkeypatch) -> None: module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") canonical = {"artifact": "teleo_cloudsql_canonical_kb_context", "hit_count_total": 0, "hits": []} monkeypatch.setattr(module, "query_canonical_rows", lambda *_args: canonical.copy()) - monkeypatch.setattr(module, "audit_restore_available", lambda _args: False) + monkeypatch.delenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", raising=False) + monkeypatch.setattr( + module, + "audit_restore_available", + lambda _args: (_ for _ in ()).throw(AssertionError("audit availability must not be probed")), + ) monkeypatch.setattr( module, "query_audit_rows", @@ -373,7 +1193,22 @@ def test_cloudsql_zero_hit_search_stays_canonical_when_audit_is_unavailable(monk result = module.query_rows(SimpleNamespace(), "unseen query", 8) assert result["hits"] == [] - assert "audit fallback unavailable" in result["canonical_fallback_reason"] + assert "audit fallback disabled" in result["canonical_fallback_reason"] + + +def test_cloudsql_zero_hit_search_uses_audit_only_when_explicitly_enabled(monkeypatch) -> None: + module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py") + canonical = {"artifact": "teleo_cloudsql_canonical_kb_context", "hit_count_total": 0, "hits": []} + audit = {"artifact": "teleo_cloudsql_memory_context", "hits": [{"row_id": "audit-1"}]} + monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "1") + monkeypatch.setattr(module, "query_canonical_rows", lambda *_args: canonical.copy()) + monkeypatch.setattr(module, "audit_restore_available", lambda _args: True) + monkeypatch.setattr(module, "query_audit_rows", lambda *_args: audit.copy()) + + result = module.query_rows(SimpleNamespace(), "unseen query", 8) + + assert result["hits"] == [{"row_id": "audit-1"}] + assert result["canonical_fallback_reason"] == "no matching canonical public/persona/strategy/belief rows" def test_vps_bridge_verifies_relative_source_artifact_against_canonical_hash(tmp_path: Path) -> None: diff --git a/tests/test_leo_negative_permissions_canary.py b/tests/test_leo_negative_permissions_canary.py new file mode 100644 index 0000000..8e5ebc7 --- /dev/null +++ b/tests/test_leo_negative_permissions_canary.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +CANARY = ROOT / "scripts" / "run_leo_negative_permissions_canary.py" + + +@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker is required") +def test_leo_negative_permissions_canary_runs_real_postgres_lifecycle(tmp_path: Path) -> None: + output = tmp_path / "negative-permissions.json" + completed = subprocess.run( + [sys.executable, str(CANARY), "--output", str(output)], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + timeout=120, + ) + + assert completed.returncode == 0, completed.stderr or completed.stdout + receipt = json.loads(output.read_text(encoding="utf-8")) + assert receipt["passed"] is True + assert receipt["current_tier"] == "T2_runtime" + assert receipt["identity_policy"] == { + "exact_handle_required": True, + "participant_handle": "m3taversal", + "reviewer_handle": "m3taversal", + } + assert receipt["scope"] == { + "canonical_production_rows_contacted": False, + "container_network": "none", + "database": "teleo_canonical", + "principal": "sa-observatory-read-adapter@teleo-501523.iam", + "production_contacted": False, + } + assert receipt["summary"] == { + "allowed_expected": 3, + "allowed_passed": 3, + "denied_expected": 16, + "denied_passed": 16, + } + + operations = {operation["id"]: operation for operation in receipt["operations"]} + assert set(operations) == { + "session_read_only_defaults", + "canonical_retrieval", + "proposal_ledger_retrieval", + "canonical_insert_default_read_only", + "canonical_insert_acl_after_read_only_override", + "canonical_update_acl_after_read_only_override", + "canonical_delete_acl_after_read_only_override", + "canonical_truncate_acl_after_read_only_override", + "staging_insert_acl_after_read_only_override", + "fake_approval_gate_call", + "apply_assert_gate_call", + "apply_finish_gate_call", + "protected_schema_ddl_after_read_only_override", + "outside_allowlist_retrieval", + "create_role_escalation", + "grant_apply_role_escalation", + "set_apply_role_escalation", + "alter_role_createdb_escalation", + "claim_table_ownership_escalation", + } + assert all(operation["passed"] for operation in operations.values()) + assert receipt["fingerprint"]["before"]["participant_handle"] == "m3taversal" + assert receipt["fingerprint"]["after"]["participant_handle"] == "m3taversal" + assert receipt["fingerprint"]["unchanged"] is True + assert receipt["fingerprint"]["before_sha256"] == receipt["fingerprint"]["after_sha256"] + assert receipt["cleanup"]["passed"] is True + assert receipt["cleanup"]["exact_name_orphans"] == [] + assert receipt["cleanup"]["label_scoped_orphans"] == [] + assert receipt["cleanup"]["temporary_workdir_exists"] is False diff --git a/tests/test_observatory_read_role_postgres.py b/tests/test_observatory_read_role_postgres.py index 97f0921..c90eee0 100644 --- a/tests/test_observatory_read_role_postgres.py +++ b/tests/test_observatory_read_role_postgres.py @@ -137,6 +137,105 @@ def test_observatory_role_is_read_only_and_exactly_allowlisted() -> None: input_text=ROLE_SQL.read_text(encoding="utf-8"), ) assert '"effective_table_writes_denied": true' in migration.stdout + assert '"effective_schema_create_denied": true' in migration.stdout + assert '"protected_apply_execute_denied": true' in migration.stdout + + run( + [ + "docker", + "exec", + container, + "psql", + "-U", + "postgres", + "-d", + DATABASE, + "-c", + "grant create on schema public to public;", + ] + ) + unsafe_schema = run( + [ + "docker", + "exec", + "-e", + f"OBSERVATORY_DB_IAM_USER={IAM_USER}", + "-i", + container, + "psql", + "-U", + "postgres", + "-d", + DATABASE, + ], + input_text=ROLE_SQL.read_text(encoding="utf-8"), + check=False, + ) + assert unsafe_schema.returncode != 0 + assert "can create objects in protected schemas: public" in unsafe_schema.stderr + run( + [ + "docker", + "exec", + container, + "psql", + "-U", + "postgres", + "-d", + DATABASE, + "-c", + "revoke create on schema public from public;", + ] + ) + + run( + [ + "docker", + "exec", + container, + "psql", + "-U", + "postgres", + "-d", + DATABASE, + "-c", + "create function kb_stage.approve_strict_proposal(uuid, text, jsonb, text, text) " + "returns jsonb language sql as $$ select '{}'::jsonb $$;", + ] + ) + unsafe_function = run( + [ + "docker", + "exec", + "-e", + f"OBSERVATORY_DB_IAM_USER={IAM_USER}", + "-i", + container, + "psql", + "-U", + "postgres", + "-d", + DATABASE, + ], + input_text=ROLE_SQL.read_text(encoding="utf-8"), + check=False, + ) + assert unsafe_function.returncode != 0 + assert "can execute protected proposal/apply gates" in unsafe_function.stderr + run( + [ + "docker", + "exec", + container, + "psql", + "-U", + "postgres", + "-d", + DATABASE, + "-c", + "drop function kb_stage.approve_strict_proposal(uuid, text, jsonb, text, text);", + ] + ) readback = run( [ diff --git a/tests/test_teleo_agent_systemd.py b/tests/test_teleo_agent_systemd.py index 154ac86..1156c9e 100644 --- a/tests/test_teleo_agent_systemd.py +++ b/tests/test_teleo_agent_systemd.py @@ -1,9 +1,33 @@ +import os +import resource +import signal import subprocess +import sys +import time from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[1] +def embedded_service_context_runner_source() -> str: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + marker = "remote_helpers=$(cat <<'REMOTE'\n" + start = script.index(marker) + len(marker) + end = script.index("\nREMOTE\n)", start) + helpers = script[start:end] + blocks = [part.split("\nPY", maxsplit=1)[0] for part in helpers.split("<<'PY'\n")[1:]] + return next(block for block in blocks if "def run_stopped_child(" in block) + + +def load_service_context_runner() -> dict[str, object]: + namespace: dict[str, object] = {"__name__": "embedded_service_context_runner_test"} + source = embedded_service_context_runner_source() + exec(compile(source, "", "exec"), namespace) + return namespace + + def test_teleo_agent_template_supports_optional_per_agent_env_file(): unit = (REPO_ROOT / "systemd" / "teleo-agent@.service").read_text() @@ -41,7 +65,7 @@ def test_auto_deploy_prefers_github_remote_when_present(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() assert 'DEPLOY_REMOTE="${TELEO_DEPLOY_REMOTE:-}"' in auto_deploy - assert 'remote get-url github' in auto_deploy + assert "remote get-url github" in auto_deploy assert 'git fetch "$DEPLOY_REMOTE" main' in auto_deploy assert 'git rev-parse "$DEPLOY_REMOTE/main"' in auto_deploy assert 'git merge --ff-only "$DEPLOY_REMOTE/main"' in auto_deploy @@ -50,7 +74,7 @@ def test_auto_deploy_prefers_github_remote_when_present(): def test_auto_deploy_executable_safety_net_stays_inside_synced_dirs(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() - assert "for dir in \"$PIPELINE_DIR\" \"$TELEGRAM_DIR\" \"$DIAGNOSTICS_DIR\" \"$AGENT_STATE_DIR\"" in auto_deploy + assert 'for dir in "$PIPELINE_DIR" "$TELEGRAM_DIR" "$DIAGNOSTICS_DIR" "$AGENT_STATE_DIR"' in auto_deploy assert "find /opt/teleo-eval -maxdepth 3 -name '*.sh'" not in auto_deploy assert "backups" in auto_deploy @@ -61,7 +85,7 @@ def test_agent_healthcheck_timer_runs_both_live_telegram_agents(): assert "/opt/teleo-eval/telegram/agent_healthcheck.py" in service assert "--agents leo leo-wallet-test" in service - assert "--since \"20 min ago\"" in service + assert '--since "20 min ago"' in service assert "OnUnitActiveSec=5min" in timer assert "WantedBy=timers.target" in timer @@ -74,7 +98,7 @@ def test_deploy_scripts_install_systemd_units_and_enable_agent_healthcheck(): assert 'sudo install -m 0644 "$unit" "$SYSTEMD_DIR/$(basename "$unit")"' in auto_deploy assert "sudo systemctl daemon-reload" in auto_deploy assert "sudo systemctl enable --now teleo-agent-healthcheck.timer" in auto_deploy - assert "git diff --name-only \"$OLD_SHA\" \"$NEW_SHA\" -- systemd/teleo-agent@.service" in auto_deploy + assert 'git diff --name-only "$OLD_SHA" "$NEW_SHA" -- systemd/teleo-agent@.service' in auto_deploy assert 'VPS_SYSTEMD="/etc/systemd/system"' in manual_deploy assert "teleo-agent-healthcheck.timer" in manual_deploy @@ -100,16 +124,16 @@ def test_deploy_scripts_sync_leoclean_skills_and_restart_gateway_for_runtime_cha assert 'LEOCLEAN_BIN_DIR="/home/teleo/.hermes/profiles/leoclean/bin"' in auto_deploy assert 'LEOCLEAN_SKILLS_DIR="/home/teleo/.hermes/profiles/leoclean/skills"' in auto_deploy assert 'LEOCLEAN_PLUGINS_DIR="/home/teleo/.hermes/profiles/leoclean/plugins"' in auto_deploy - assert 'hermes-agent/leoclean-bin/' in auto_deploy - assert 'hermes-agent/leoclean-skills/vps/' in auto_deploy - assert 'hermes-agent/leoclean-plugins/vps/' in auto_deploy + assert "hermes-agent/leoclean-bin/" in auto_deploy + assert "hermes-agent/leoclean-skills/vps/" in auto_deploy + assert "hermes-agent/leoclean-plugins/vps/" in auto_deploy assert 'HERMES_PATCH_DIR="/home/teleo/.hermes/teleo-runtime-patches"' in auto_deploy assert '"status": "installed_now"' in auto_deploy - assert 'Hermes response-transform drift repaired' in auto_deploy - assert 'deploy/leoclean-gateway-restart-required.sh' in auto_deploy - assert 'TELEO_AUTO_DEPLOY_REEXECED' in auto_deploy + assert "Hermes response-transform drift repaired" in auto_deploy + assert "deploy/leoclean-gateway-restart-required.sh" in auto_deploy + assert "TELEO_AUTO_DEPLOY_REEXECED" in auto_deploy assert 'exec bash "$DEPLOY_CHECKOUT/deploy/auto-deploy.sh"' in auto_deploy - assert 'add_restart_if_unit_active leoclean-gateway' in auto_deploy + assert "add_restart_if_unit_active leoclean-gateway" in auto_deploy assert 'sudo systemctl restart "$svc"' in auto_deploy assert "sudo systemctl restart $RESTART" not in auto_deploy assert "NOPASSWD: /usr/bin/systemctl restart leoclean-gateway" in sudoers @@ -117,11 +141,11 @@ def test_deploy_scripts_sync_leoclean_skills_and_restart_gateway_for_runtime_cha assert 'VPS_LEOCLEAN_BIN="/home/teleo/.hermes/profiles/leoclean/bin"' in manual_deploy assert 'VPS_LEOCLEAN_SKILLS="/home/teleo/.hermes/profiles/leoclean/skills"' in manual_deploy assert 'VPS_LEOCLEAN_PLUGINS="/home/teleo/.hermes/profiles/leoclean/plugins"' in manual_deploy - assert 'hermes-agent/leoclean-bin/' in manual_deploy - assert 'hermes-agent/leoclean-skills/vps/' in manual_deploy - assert 'hermes-agent/leoclean-plugins/vps/' in manual_deploy + assert "hermes-agent/leoclean-bin/" in manual_deploy + assert "hermes-agent/leoclean-skills/vps/" in manual_deploy + assert "hermes-agent/leoclean-plugins/vps/" in manual_deploy assert 'VPS_HERMES_PATCHES="/home/teleo/.hermes/teleo-runtime-patches"' in manual_deploy - assert 'systemctl is-active --quiet leoclean-gateway.service' in manual_deploy + assert "systemctl is-active --quiet leoclean-gateway.service" in manual_deploy def test_deploy_scripts_syntax_check_leoclean_database_runtime_before_restart(): @@ -135,32 +159,22 @@ def test_deploy_scripts_syntax_check_leoclean_database_runtime_before_restart(): def test_auto_deploy_hot_syncs_leoclean_markdown_without_gateway_restart(): - assert not _leoclean_gateway_restart_required( - "hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md" - ) - assert not _leoclean_gateway_restart_required( - "hermes-agent/leoclean-skills/vps/live-leo-telegram/SKILL.md" - ) + assert not _leoclean_gateway_restart_required("hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md") + assert not _leoclean_gateway_restart_required("hermes-agent/leoclean-skills/vps/live-leo-telegram/SKILL.md") def test_auto_deploy_restarts_gateway_for_leoclean_runtime_changes(): assert _leoclean_gateway_restart_required("hermes-agent/leoclean-bin/kb_tool.py") - assert _leoclean_gateway_restart_required( - "hermes-agent/leoclean-skills/vps/teleo-kb-bridge/runtime_helper.py" - ) - assert _leoclean_gateway_restart_required( - "hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py" - ) - assert _leoclean_gateway_restart_required( - "hermes-agent/patches/apply_response_transform_hook.py" - ) + assert _leoclean_gateway_restart_required("hermes-agent/leoclean-skills/vps/teleo-kb-bridge/runtime_helper.py") + assert _leoclean_gateway_restart_required("hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py") + assert _leoclean_gateway_restart_required("hermes-agent/patches/apply_response_transform_hook.py") def test_auto_deploy_sudoers_installer_is_root_only_and_visudo_checked(): installer = (REPO_ROOT / "deploy" / "install-auto-deploy-sudoers.sh").read_text() assert 'TARGET="/etc/sudoers.d/teleo-auto-deploy"' in installer - assert 'id -u' in installer + assert "id -u" in installer assert 'install -m 0440 "$SOURCE" "$TARGET"' in installer assert 'visudo -cf "$TARGET"' in installer @@ -183,3 +197,672 @@ def test_gcp_leoclean_skill_sync_targets_parallel_runtime_without_secrets(): assert "sudo rsync" not in script assert "TELEGRAM_BOT_TOKEN" not in script assert "CLOUDSQL_PASSWORD" not in script + + +def test_gcp_runtime_sync_is_source_bound_and_scoped(): + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + dropin = (REPO_ROOT / "systemd" / "leoclean-gcp-prod-parallel-cloudsql.conf").read_text() + + assert 'git -C "$REPO_ROOT" rev-parse HEAD' in script + assert 'git -C "$REPO_ROOT" diff --quiet HEAD' in script + assert "merge-base --is-ancestor" in script + assert "Refusing to deploy unmerged revision" in script + assert "Refusing to install runtime files without --restart" in script + assert 'DROPIN_NAME="10-cloudsql-memory.conf"' in script + assert "20-canonical-kb.conf" not in script + assert "sha256sum" in script + assert "--preflight-only" in script + assert "--restart" in script + assert '/usr/bin/python3 -I "$tool_path" status --format json --redacted' in script + assert "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" in script + assert "169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" in script + assert "ProxyHandler({})" in script + assert "env -i" in script + assert 'run_scoped_status pre "$payload_dir/$TOOL_REL"' in script + assert 'run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL"' in script + assert 'sudo -n -u teleo mkdir -p -- "$receipt_dir"' in script + assert 'if [ "$PREFLIGHT_ONLY" = true ]; then' in script + assert "run_service_wrapper_status post" in script + assert "run_service_wrapper_status verify" in script + assert '"$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted' in script + wrapper_probe = script.split("run_service_wrapper_status() {", maxsplit=1)[1].split( + "\n}\n\nservice_fingerprint", maxsplit=1 + )[0] + assert 'run_in_service_cgroup "$main_pid"' in wrapper_probe + assert 'nsenter -t "$main_pid" -m -n --env' in wrapper_probe + assert "/usr/bin/setpriv" in wrapper_probe + assert "--reuid=teleo" in wrapper_probe + assert "TELEO_CLOUDSQL_" not in wrapper_probe + assert "/usr/bin/env -i" not in wrapper_probe + assert 'run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py"' in script + assert 'SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py"' in script + assert 'assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py"' in script + assert 'assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"' in script + assert "REMOTE_CLOUDSDK_CONFIG" not in script + assert "--property=DropInPaths" in script + assert "--property=EnvironmentFiles" in script + assert "Service has an effective EnvironmentFile" in script + assert "Service has a later drop-in" in script + assert "--property=PrivateNetwork" in script + assert "--property=PrivateUsers" in script + assert "NetworkNamespacePath" in script + assert "IPAddressDeny" in script + assert "IPAddressAllow" in script + assert "RestrictAddressFamilies" in script + assert "RestrictNetworkInterfaces" in script + assert "SystemCallFilter" in script + assert "AppArmorProfile" in script + assert "SELinuxContext" in script + assert '--property="$network_property"' in script + assert "assert_administrator_secret_denied" in script + assert '"stdout_discarded": True' in script + assert '"credential_context": "systemd_main_pid_environment"' in script + administrator_probe = script.split("assert_administrator_secret_denied() {", maxsplit=1)[1].split( + "\n}\n\nrun_scoped_status", maxsplit=1 + )[0] + assert 'run_in_service_cgroup "$main_pid"' in administrator_probe + assert 'nsenter -t "$main_pid" -m -n --env' in administrator_probe + assert "/usr/bin/setpriv" in administrator_probe + assert "env -i" not in administrator_probe + permission_probe = script.split("run_permission_verifier() {", maxsplit=1)[1].split( + "\n}\n\nassert_trusted_executable", maxsplit=1 + )[0] + assert 'if [ "$phase" = pre ]; then' in permission_probe + assert 'run_in_service_cgroup "$main_pid"' in permission_probe + assert 'nsenter -t "$main_pid" -m -n' in permission_probe + assert "/usr/bin/env -i" in permission_probe + assert 'verification_tmp="$(sudo mktemp -d /run/' in script + assert "run_in_service_cgroup() {" in script + assert "--property=ControlGroup" in script + assert 'Path(f"/proc/{pid}/cgroup")' in script + assert 'Path("/sys/fs/cgroup").resolve(strict=True)' in script + assert 'cgroup_dir / "cgroup.procs"' in script + assert "os.WUNTRACED" in script + assert "signal.SIGSTOP" in script + assert "signal.SIGCONT" in script + assert "os.waitpid(child, 0)" in script + assert "PR_SET_PDEATHSIG" in script + assert "signal.pthread_sigmask" in script + assert "ParentInterrupted" in script + assert "kill_and_reap(child)" in script + assert "Seccomp_filters" in script + assert '"/proc/$main_pid/attr/current"' in script + assert "/proc/1/ns/user" in script + assert "resource.prlimit" in script + assert "copy_process_limits" in script + assert "configure_stopped_child" in script + assert 'preflight_service_before="$(service_fingerprint)"' in script + assert 'assert_service_fingerprint "$preflight_service_before"' in script + assert "backup_path" in script + assert "backup_directory_state" in script + assert "restore_path" in script + assert "restore_directory_state" in script + assert 'runtime_grandparent="$(dirname "$runtime_parent")"' in script + assert 'runtime_anchor="$(dirname "$runtime_grandparent")"' in script + assert 'dropin_parent="$(dirname "$dropin_dir")"' in script + assert 'backup_directory_state "$runtime_grandparent" runtime_grandparent' in script + assert 'restore_directory_state "$runtime_grandparent" runtime_grandparent' in script + assert 'sudo test -d "$runtime_anchor"' in script + assert 'sudo test -d "$dropin_parent"' in script + assert 'sudo rmdir -- "$target"' in script + assert "rollback_runtime" in script + assert "trap 'on_exit $?' EXIT" in script + assert 'backup_path "$dropin_dir/$DROPIN_NAME" dropin' in script + assert 'dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$"' in script + assert 'sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME"' in script + assert 'active_state" != "active"' in script + assert 'sub_state" != "running"' in script + assert 'sudo systemctl stop "$SERVICE"' in script + assert 'sudo systemctl start "$SERVICE"' in script + assert 'REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"' in script + assert 'sudo install -d -o root -g root -m 0755 "$runtime_parent"' in script + assert 'sudo install -d -o root -g root -m 0755 "$runtime_bin"' in script + assert 'remote_tmp="$(sudo mktemp -d /run/teleo-gcp-leoclean-runtime.XXXXXX)"' in script + assert 'sudo install -d -o root -g root -m 0755 "$payload_dir"' in script + assert 'sudo install -d -o root -g root -m 0700 "$backup_dir"' in script + assert 'sudo install -d -o teleo -g teleo -m 0700 "$work_dir"' in script + assert "sudo tar --no-same-owner --no-same-permissions" in script + assert 'sudo chown -R root:root "$payload_dir"' in script + assert 'sudo find "$payload_dir" -type d -exec chmod 0555 {} +' in script + assert 'sudo find "$payload_dir" -type l -print -quit' in script + assert 'sudo -n -u teleo test ! -w "$payload_dir"' in script + assert 'sudo -n -u teleo test ! -w "$backup_dir"' in script + assert 'sudo touch "$backup_dir/$name.present"' in script + assert 'sudo test -f "$backup_dir/$name.present"' in script + assert 'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage"' in script + assert 'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage"' in script + assert ( + 'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" ' + '"$permission_verifier_stage"' in script + ) + assert ( + 'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" ' + '"$service_env_verifier_stage"' in script + ) + assert "assert_installed_files\nsudo systemctl daemon-reload" in script + assert 'tar -C "$remote_tmp" -xf -' not in script + assert 'mkdir -p "$backup_dir"' not in script + assert 'for protected_file in \\\n "$wrapper" \\' in script + assert '"$permission_verifier" \\' in script + assert '"$service_env_verifier" \\' in script + assert 'sudo -n -u teleo test ! -w "$protected_file"' in script + assert "for protected_dir in \\\n /usr \\" in script + assert "/usr/local/libexec/livingip" in script + assert 'sudo -n -u teleo test ! -w "$protected_dir"' in script + assert "for executable_dir in \\\n /usr/local/sbin \\" in script + assert "sudo stat -Lc '%U:%G' \"$executable_dir\"" in script + assert 'sudo -n -u teleo test ! -w "$executable_dir"' in script + assert ( + "for protected_file in /bin/bash /usr/bin/curl /usr/bin/env /usr/bin/gcloud " + "/usr/bin/psql /usr/bin/python3 /usr/bin/setpriv" in script + ) + assert "/usr/bin/curl -q --noproxy" in script + assert 'test "$(sudo cat "$revision")" = "$SOURCE_REVISION"' in script + assert 'assert_existing_root_directory "$existing_directory"' in script + assert 'assert_existing_root_file "$existing_file"' in script + assert 'validate_restored_path "$runtime_bin/teleo-kb" wrapper' in script + assert 'validate_restored_directory "$runtime_bin" runtime_bin' in script + rollback = script.split("rollback_runtime() {", maxsplit=1)[1].split("\n}\n\non_exit()", maxsplit=1)[0] + assert rollback.index("validate_restored_path") < rollback.index("systemctl daemon-reload") + assert rollback.index("systemctl daemon-reload") < rollback.index('systemctl start "$SERVICE"') + assert rollback.count('if [ "$rollback_rc" -eq 0 ]; then') >= 3 + deploy_tail = script.split("mutation_started=true", maxsplit=1)[1] + assert deploy_tail.index("assert_installed_files") < deploy_tail.index("systemctl daemon-reload") + assert deploy_tail.index("systemctl daemon-reload") < deploy_tail.index("assert_unit_configuration") + assert deploy_tail.index("assert_unit_configuration") < deploy_tail.index('systemctl start "$SERVICE"') + for install_line in ( + 'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL"', + 'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL"', + 'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL"', + 'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL"', + 'sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL"', + ): + assert install_line in deploy_tail + assert 'sudo install -o root -g root -m 0755 "$work_dir/' not in script + assert 'nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb"' in script + assert 'findmnt -T "$SERVICE_PROFILE_BIN"' in script + assert "findmnt -M /home/teleo" in script + assert 'findmnt -M "$SERVICE_PROFILE_ROOT/state"' in script + assert 'findmnt -M "$SERVICE_PROFILE_ROOT/workspace"' in script + assert "for capability_field in CapInh CapPrm CapEff CapBnd CapAmb" in script + assert "0000000000000000" in script + assert '"current_tier": "T3_live_readonly"' in script + assert '"status": "pass"' in script + assert "UnsetEnvironment=PGPASSWORD" in dropin + assert "UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN" in dropin + assert "UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS" in dropin + assert "UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT" in dropin + assert "UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT" in dropin + assert "UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY" in dropin + assert "UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE" in dropin + assert ( + "UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL " + "TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT" in dropin + ) + assert ( + "BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin" in dropin + ) + assert "ReadOnlyPaths=/home/teleo" in dropin + assert ( + "ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state " + "/home/teleo/.hermes/profiles/leoclean/workspace" in dropin + ) + assert "CapabilityBoundingSet=" in dropin + assert "AmbientCapabilities=" in dropin + assert "RuntimeDirectory=" not in dropin + assert "Environment=CLOUDSDK_CONFIG=/usr/local/libexec/livingip/leoclean-kb/gcloud-config" in dropin + assert "Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" in dropin + assert "TELEO_KB_MODE=cloudsql" in dropin + assert "TELEO_GCP_METADATA_ONLY=1" in dropin + assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in dropin + assert "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" in dropin + assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in dropin + assert not any(line.startswith("Environment=PGPASSWORD") for line in dropin.splitlines()) + assert "gcp-teleo-pgvector-standby-postgres-password" not in dropin + + +def test_gcp_runtime_embedded_remote_programs_are_bash_syntax_valid() -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + blocks: dict[str, str] = {} + + for variable in ("remote_helpers", "verify_body", "sync_body"): + marker = f"{variable}=$(cat <<'REMOTE'\n" + assert script.count(marker) == 1 + start = script.index(marker) + len(marker) + end = script.index("\nREMOTE\n)", start) + blocks[variable] = script[start:end] + + for name, body in ( + *blocks.items(), + ("composed_verify", blocks["remote_helpers"] + "\n" + blocks["verify_body"]), + ("composed_sync", blocks["remote_helpers"] + "\n" + blocks["sync_body"]), + ): + completed = subprocess.run( + ["bash", "-n"], + input=body, + capture_output=True, + check=False, + text=True, + ) + assert completed.returncode == 0, f"{name}: {completed.stderr}" + + embedded_python = [part.split("\nPY", maxsplit=1)[0] for part in blocks["remote_helpers"].split("<<'PY'\n")[1:]] + assert len(embedded_python) == 4 + for index, body in enumerate(embedded_python): + completed = subprocess.run( + ["python3", "-c", "import sys; compile(sys.stdin.read(), '', 'exec')"], + input=body, + capture_output=True, + check=False, + text=True, + ) + assert completed.returncode == 0, f"embedded_python_{index}: {completed.stderr}" + + +@pytest.mark.parametrize( + ("command", "expected_returncode"), + [ + (["/bin/sh", "-c", "exit 0"], 0), + (["/bin/sh", "-c", "exit 7"], 7), + (["/bin/sh", "-c", "kill -TERM $$"], 128 + signal.SIGTERM), + ], +) +def test_service_context_runner_propagates_result_and_reaps_child( + command: list[str], + expected_returncode: int, +) -> None: + run_stopped_child = load_service_context_runner()["run_stopped_child"] + moved_children: list[int] = [] + + result = run_stopped_child( # type: ignore[operator] + command, + "/test.service", + move_child=moved_children.append, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + ) + + assert result == expected_returncode + assert len(moved_children) == 1 + with pytest.raises(ProcessLookupError): + os.kill(moved_children[0], 0) + + +def test_service_context_runner_reaps_stopped_child_when_move_fails() -> None: + run_stopped_child = load_service_context_runner()["run_stopped_child"] + stopped_children: list[int] = [] + + def fail_move(child: int) -> None: + stopped_children.append(child) + raise RuntimeError("synthetic move failure") + + with pytest.raises(RuntimeError, match="synthetic move failure"): + run_stopped_child( # type: ignore[operator] + ["/bin/sh", "-c", "exit 0"], + "/test.service", + move_child=fail_move, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + ) + + assert len(stopped_children) == 1 + with pytest.raises(ProcessLookupError): + os.kill(stopped_children[0], 0) + + +@pytest.mark.skipif(not hasattr(resource, "prlimit"), reason="requires Linux prlimit") +def test_service_context_runner_copies_mainpid_resource_limits(tmp_path: Path) -> None: + namespace = load_service_context_runner() + run_stopped_child = namespace["run_stopped_child"] + copy_process_limits = namespace["copy_process_limits"] + limit_readback = tmp_path / "limits.txt" + child_program = ( + "import resource,sys; from pathlib import Path; " + "limits=resource.getrlimit(resource.RLIMIT_NOFILE); " + "Path(sys.argv[1]).write_text(f'{limits[0]}:{limits[1]}', encoding='ascii')" + ) + + def lower_nofile_limit() -> None: + resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) + + source = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)"], + preexec_fn=lower_nofile_limit, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + result = run_stopped_child( # type: ignore[operator] + [sys.executable, "-c", child_program, str(limit_readback)], + "/test.service", + move_child=lambda _child: None, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + configure_stopped_child=lambda child: copy_process_limits(source.pid, child), # type: ignore[operator] + ) + finally: + source.terminate() + source.wait(timeout=5) + + assert result == 0 + assert limit_readback.read_text(encoding="ascii") == "64:64" + + +@pytest.mark.parametrize("signum", [signal.SIGTERM, signal.SIGHUP]) +def test_service_context_runner_interruption_kills_and_reaps_probe( + tmp_path: Path, + signum: signal.Signals, +) -> None: + run_stopped_child = load_service_context_runner()["run_stopped_child"] + child_pid_path = tmp_path / f"probe-{signum.name}.pid" + child_program = ( + "import os,sys,time; from pathlib import Path; " + "Path(sys.argv[1]).write_text(str(os.getpid()), encoding='ascii'); " + "time.sleep(30)" + ) + runner_pid = os.fork() + if runner_pid == 0: + result = run_stopped_child( # type: ignore[operator] + [sys.executable, "-c", child_program, str(child_pid_path)], + "/test.service", + move_child=lambda _child: None, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + ) + os._exit(result) + + runner_reaped = False + probe_pid: int | None = None + try: + deadline = time.monotonic() + 5 + probe_pid_text = "" + while time.monotonic() < deadline: + if child_pid_path.is_file(): + probe_pid_text = child_pid_path.read_text(encoding="ascii") + if probe_pid_text.isdecimal(): + break + waited, _status = os.waitpid(runner_pid, os.WNOHANG) + if waited == runner_pid: + runner_reaped = True + pytest.fail("service-context runner exited before probe start") + time.sleep(0.01) + assert probe_pid_text.isdecimal() + probe_pid = int(probe_pid_text) + + os.kill(runner_pid, signum) + waited, status = os.waitpid(runner_pid, 0) + runner_reaped = True + assert waited == runner_pid + assert os.WIFSIGNALED(status) + assert os.WTERMSIG(status) == signum + with pytest.raises(ProcessLookupError): + os.kill(probe_pid, 0) + finally: + if not runner_reaped: + try: + os.kill(runner_pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.waitpid(runner_pid, 0) + except ChildProcessError: + pass + if probe_pid is not None: + try: + os.kill(probe_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def test_gcp_runtime_rejects_preexisting_symlink_before_mutation(tmp_path: Path) -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("assert_existing_root_directory() {") + function_end = script.index("\n\nbackup_path() {", function_start) + trust_functions = script[function_start:function_end] + real_directory = tmp_path / "real-runtime" + real_directory.mkdir() + symlink = tmp_path / "runtime-link" + symlink.symlink_to(real_directory, target_is_directory=True) + harness = f""" +set -euo pipefail +sudo() {{ + if [ "${{1:-}}" = -n ]; then + shift + if [ "${{1:-}}" = -u ]; then shift 2; fi + fi + "$@" +}} +{trust_functions} +assert_existing_root_directory "$1" +""" + completed = subprocess.run( + ["bash", "-s", "--", str(symlink)], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode != 0 + + +def test_gcp_runtime_executable_configuration_override_gate() -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("assert_unit_configuration() {") + function_end = script.index("\n\nassert_runtime_environment() {", function_start) + unit_configuration_function = script[function_start:function_end] + harness = f""" +set -euo pipefail +SERVICE=leoclean.service +REMOTE_DROPIN_DIR=/etc/systemd/system/leoclean.service.d +DROPIN_NAME=10-cloudsql-memory.conf +REMOTE_RUNTIME_BIN=/usr/local/libexec/livingip/leoclean-kb +SERVICE_PROFILE_ROOT=/home/teleo/.hermes/profiles/leoclean +SERVICE_PROFILE_BIN=$SERVICE_PROFILE_ROOT/bin +assert_exact_word_set() {{ + local actual="$1" expected="$2" label="$3" actual_sorted expected_sorted word + actual_sorted="$(for word in $actual; do printf '%s\\n' "${{word#-}}"; done | sort)" + expected_sorted="$(for word in $expected; do printf '%s\\n' "${{word#-}}"; done | sort)" + [ "$actual_sorted" = "$expected_sorted" ] || {{ + echo "Service effective $label differs from the reviewed exact allowlist." >&2 + return 1 + }} +}} +systemctl() {{ + local argument property= + for argument in "$@"; do + case "$argument" in --property=*) property=${{argument#--property=}} ;; esac + done + case "$property" in + DropInPaths) printf '%s\\n' "${{FAKE_DROPINS:-}}" ;; + EnvironmentFiles) printf '%s\\n' "${{FAKE_ENVIRONMENT_FILES:-}}" ;; + LoadCredential|LoadCredentialEncrypted|SetCredential|SetCredentialEncrypted|ImportCredential|ExecCondition|ExecStartPre|ExecStartPost|ExecReload|ExecStop|ExecStopPost) + local variable="FAKE_$property" + printf '%s\\n' "${{!variable:-}}" + ;; + CapabilityBoundingSet|AmbientCapabilities|BindPaths|TemporaryFileSystem) printf '\\n' ;; + NoNewPrivileges) printf 'yes\\n' ;; + PrivateNetwork) printf '%s\\n' "${{FAKE_PrivateNetwork:-no}}" ;; + PrivateUsers) printf '%s\\n' "${{FAKE_PrivateUsers:-no}}" ;; + NetworkNamespacePath|IPAddressDeny|IPAddressAllow|RestrictAddressFamilies|RestrictNetworkInterfaces|SocketBindAllow|SocketBindDeny|SystemCallFilter|AppArmorProfile|SELinuxContext) + local variable="FAKE_$property" + printf '%s\\n' "${{!variable:-}}" + ;; + Group|SupplementaryGroups) printf 'teleo\\n' ;; + ReadOnlyPaths) printf '%s\\n' "${{FAKE_ReadOnlyPaths:-/home/teleo}}" ;; + ReadWritePaths) printf '%s\\n' "$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" ;; + BindReadOnlyPaths) printf '%s\\n' "$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" ;; + InaccessiblePaths) printf '%s\\n' '/home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql' ;; + *) return 90 ;; + esac +}} +{unit_configuration_function} +assert_unit_configuration +""" + reviewed = "/etc/systemd/system/leoclean.service.d/10-cloudsql-memory.conf" + cases = ( + { + "FAKE_DROPINS": f"{reviewed} /etc/systemd/system/leoclean.service.d/99-conflict.conf", + "FAKE_ENVIRONMENT_FILES": "", + }, + {"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": "/etc/leoclean-runtime.env"}, + {"FAKE_DROPINS": reviewed, "FAKE_ExecStartPost": "/usr/local/bin/conflicting-hook"}, + {"FAKE_DROPINS": reviewed, "FAKE_ReadOnlyPaths": "/home/teleo /tmp"}, + {"FAKE_DROPINS": reviewed, "FAKE_PrivateNetwork": "yes"}, + {"FAKE_DROPINS": reviewed, "FAKE_PrivateUsers": "yes"}, + {"FAKE_DROPINS": reviewed, "FAKE_NetworkNamespacePath": "/run/netns/restricted"}, + {"FAKE_DROPINS": reviewed, "FAKE_IPAddressDeny": "any"}, + {"FAKE_DROPINS": reviewed, "FAKE_IPAddressAllow": "10.61.0.4/32"}, + {"FAKE_DROPINS": reviewed, "FAKE_RestrictAddressFamilies": "AF_UNIX"}, + {"FAKE_DROPINS": reviewed, "FAKE_RestrictNetworkInterfaces": "~ens4"}, + {"FAKE_DROPINS": reviewed, "FAKE_SocketBindDeny": "any"}, + {"FAKE_DROPINS": reviewed, "FAKE_SystemCallFilter": "@system-service ~@network-io"}, + {"FAKE_DROPINS": reviewed, "FAKE_AppArmorProfile": "restricted-profile"}, + {"FAKE_DROPINS": reviewed, "FAKE_SELinuxContext": "system_u:system_r:restricted_t:s0"}, + ) + for environment in cases: + completed = subprocess.run( + ["bash", "-s"], + input=harness, + capture_output=True, + check=False, + env=environment, + text=True, + ) + assert completed.returncode != 0 + + allowed = subprocess.run( + ["bash", "-s"], + input=harness, + capture_output=True, + check=False, + env={"FAKE_DROPINS": reviewed}, + text=True, + ) + assert allowed.returncode == 0, allowed.stderr + + +def test_gcp_runtime_effective_execstart_override_is_rejected_before_process_probe() -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("assert_service_running() {") + function_end = script.index("\n\nassert_unit_configuration() {", function_start) + service_function = script[function_start:function_end] + harness = f""" +set -euo pipefail +SERVICE=leoclean.service +EXPECTED_HERMES_EXECUTABLE=/home/teleo/.hermes/hermes-agent/hermes.py +EXPECTED_HERMES_PYTHON=/home/teleo/.hermes/hermes-agent/venv/bin/python +systemctl() {{ + local argument property= + for argument in "$@"; do + case "$argument" in --property=*) property=${{argument#--property=}} ;; esac + done + case "$property" in + ActiveState) printf 'active\\n' ;; + SubState) printf 'running\\n' ;; + MainPID) printf '1234\\n' ;; + User) printf 'teleo\\n' ;; + ExecStart) printf '{{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /tmp/conflicting.py ; }}\\n' ;; + *) return 90 ;; + esac +}} +ps() {{ printf 'teleo\\n'; }} +{service_function} +assert_service_running false +""" + + completed = subprocess.run( + ["bash", "-s"], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode != 0 + assert "effective ExecStart is not the reviewed" in completed.stderr + + +def test_gcp_runtime_absence_rollback_removes_every_created_directory(tmp_path: Path) -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("backup_directory_state() {") + function_end = script.index("\nrestore_path() {", function_start) + directory_functions = script[function_start:function_end] + harness = f""" +set -euo pipefail +sudo() {{ "$@"; }} +root="$1" +backup_dir="$root/backup" +mkdir -p "$backup_dir" "$root/usr/local" "$root/etc/systemd/system" +runtime_grandparent="$root/usr/local/libexec" +runtime_parent="$runtime_grandparent/livingip" +runtime_bin="$runtime_parent/leoclean-kb" +dropin_dir="$root/etc/systemd/system/service.d" +{directory_functions} +backup_directory_state "$runtime_grandparent" runtime_grandparent +backup_directory_state "$runtime_parent" runtime_parent +backup_directory_state "$runtime_bin" runtime_bin +backup_directory_state "$dropin_dir" dropin_dir +mkdir -p "$runtime_bin" "$dropin_dir" +restore_directory_state "$runtime_bin" runtime_bin +restore_directory_state "$runtime_parent" runtime_parent +restore_directory_state "$runtime_grandparent" runtime_grandparent +restore_directory_state "$dropin_dir" dropin_dir +test ! -e "$runtime_grandparent" +test ! -e "$dropin_dir" +test -d "$root/usr/local" +test -d "$root/etc/systemd/system" +""" + completed = subprocess.run( + ["bash", "-s", "--", str(tmp_path)], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + + +def test_gcp_runtime_rollback_never_reloads_or_restarts_after_validation_failure(tmp_path: Path) -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("rollback_runtime() {") + function_end = script.index("\n\non_exit() {", function_start) + rollback_function = script[function_start:function_end] + action_log = tmp_path / "systemctl-actions" + harness = f""" +set -u +SERVICE=leoclean.service +runtime_bin=/usr/local/libexec/livingip/leoclean-kb +runtime_parent=/usr/local/libexec/livingip +runtime_grandparent=/usr/local/libexec +dropin_dir=/etc/systemd/system/leoclean.service.d +DROPIN_NAME=10-cloudsql-memory.conf +action_log="$1" +sudo() {{ + if [ "${{1:-}}" = systemctl ]; then + printf '%s\\n' "$2" >>"$action_log" + return 0 + fi + "$@" +}} +restore_path() {{ return 0; }} +restore_directory_state() {{ return 0; }} +validate_restored_path() {{ [ "$2" != wrapper ]; }} +validate_restored_directory() {{ return 0; }} +assert_unit_configuration() {{ printf '%s\\n' unit-configuration >>"$action_log"; }} +assert_service_running() {{ printf '%s\\n' assert-running >>"$action_log"; }} +{rollback_function} +rollback_runtime +""" + completed = subprocess.run( + ["bash", "-s", "--", str(action_log)], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode != 0 + assert action_log.read_text(encoding="utf-8").splitlines() == ["stop"]