From 7b16c6d3cc0fc5284526b11f16878bec112eb3d0 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 07:16:40 +0200 Subject: [PATCH] fix: prove least-privilege GCP Leo runtime context --- deploy/sync-gcp-leoclean-runtime.sh | 681 ++++++++++++-- docs/gcp-leoclean-runtime-reconciliation.md | 111 ++- .../leoclean-bin/cloudsql_memory_tool.py | 94 +- hermes-agent/leoclean-bin/teleo-kb-gcp | 52 ++ ops/gcp-teleo-pgvector-standby-server-ca.pem | 21 + ops/gcp_leoclean_runtime_role.sql | 848 ++++++++++++++---- ops/provision_gcp_leoclean_runtime_role.sh | 150 ++++ ...verify_gcp_leoclean_runtime_permissions.py | 699 +++++++++++---- ...verify_gcp_leoclean_service_environment.py | 66 +- ...run_gcp_generated_db_direct_claim_suite.py | 29 +- .../run_gcp_generated_db_working_leo_suite.py | 28 +- .../leoclean-gcp-prod-parallel-cloudsql.conf | 25 +- ...est_gcp_generated_db_direct_claim_suite.py | 8 +- ...test_gcp_generated_db_working_leo_suite.py | 8 + .../test_gcp_leoclean_runtime_permissions.py | 95 +- .../test_gcp_leoclean_service_environment.py | 6 +- .../test_hermes_leoclean_kb_bridge_source.py | 104 ++- tests/test_teleo_agent_systemd.py | 351 +++++++- 18 files changed, 2840 insertions(+), 536 deletions(-) create mode 100755 hermes-agent/leoclean-bin/teleo-kb-gcp create mode 100644 ops/gcp-teleo-pgvector-standby-server-ca.pem create mode 100755 ops/provision_gcp_leoclean_runtime_role.sh diff --git a/deploy/sync-gcp-leoclean-runtime.sh b/deploy/sync-gcp-leoclean-runtime.sh index 25a7976..b0edacb 100755 --- a/deploy/sync-gcp-leoclean-runtime.sh +++ b/deploy/sync-gcp-leoclean-runtime.sh @@ -16,11 +16,14 @@ SERVICE_PROFILE_ROOT="${SERVICE_PROFILE_ROOT:-/home/teleo/.hermes/profiles/leocl SERVICE_HERMES_ROOT="${SERVICE_HERMES_ROOT:-/home/teleo/.hermes}" REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}" EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}" +EXPECTED_HERMES_EXECUTABLE="${EXPECTED_HERMES_EXECUTABLE:-/home/teleo/.hermes/hermes-agent/venv/bin/hermes}" +EXPECTED_HERMES_PYTHON="${EXPECTED_HERMES_PYTHON:-/home/teleo/.hermes/hermes-agent/venv/bin/python3}" -WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb" +WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb-gcp" TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py" PERMISSION_VERIFIER_REL="ops/verify_gcp_leoclean_runtime_permissions.py" SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py" +SERVER_CA_REL="ops/gcp-teleo-pgvector-standby-server-ca.pem" DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf" DROPIN_NAME="10-cloudsql-memory.conf" @@ -70,6 +73,7 @@ for path in \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ "$DROPIN_REL"; do if [ ! -f "$REPO_ROOT/$path" ]; then echo "Required source file is missing: $path" >&2 @@ -93,16 +97,37 @@ if ! grep -Fxq "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$RE echo "Runtime drop-in does not bind the root-controlled runtime into the service profile." >&2 exit 65 fi -if ! grep -Fxq "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$REPO_ROOT/$DROPIN_REL"; then - echo "Runtime drop-in does not protect the service-profile ancestor from replacement." >&2 +if ! grep -Fxq 'ReadOnlyPaths=/home/teleo' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in does not make the complete service home read-only." >&2 exit 65 fi if ! grep -Fxq "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in does not limit service-profile writes to state and workspace." >&2 exit 65 fi -if ! grep -Fxq 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$REPO_ROOT/$DROPIN_REL"; then - echo "Runtime drop-in must remove CAP_SYS_ADMIN from the service." >&2 +if ! grep -Fxq 'CapabilityBoundingSet=' "$REPO_ROOT/$DROPIN_REL" || + ! grep -Fxq 'AmbientCapabilities=' "$REPO_ROOT/$DROPIN_REL" || + ! grep -Fxq 'NoNewPrivileges=yes' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must remove all capabilities and enforce no-new-privileges." >&2 + exit 65 +fi +if ! grep -Fxq 'Group=teleo' "$REPO_ROOT/$DROPIN_REL" || + ! grep -Fxq 'SupplementaryGroups=teleo' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must bind the exact service group set." >&2 + exit 65 +fi +if ! grep -Fxq 'UnsetEnvironment=OPENSSL_CONF OPENSSL_MODULES OPENSSL_ENGINES GCONV_PATH CREDENTIALS_DIRECTORY' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must remove startup-loader and systemd-credential injection fields." >&2 + exit 65 +fi +if ! grep -Fxq 'InaccessiblePaths=-/home/teleo/.config/gcloud -/home/teleo/.pgpass -/home/teleo/.pg_service.conf -/home/teleo/.postgresql' "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must mask default GCP and PostgreSQL credential paths." >&2 + exit 65 +fi +if ! grep -Fxq \ + 'UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT' \ + "$REPO_ROOT/$DROPIN_REL"; then + echo "Runtime drop-in must unset optional Cloud SQL and memory controls." >&2 exit 65 fi @@ -112,6 +137,7 @@ if ! git -C "$REPO_ROOT" diff --quiet HEAD -- \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ "$DROPIN_REL"; then echo "Refusing to deploy runtime files that do not match HEAD." >&2 exit 65 @@ -121,6 +147,7 @@ if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ "$DROPIN_REL")" ]; then echo "Refusing to deploy untracked runtime files." >&2 exit 65 @@ -137,6 +164,7 @@ WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')" TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')" PERMISSION_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" SERVICE_ENV_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" +SERVER_CA_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVER_CA_REL" | awk '{print $1}')" DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')" GCP_SSH=( @@ -147,8 +175,22 @@ GCP_SSH=( ) remote_helpers=$(cat <<'REMOTE' +assert_exact_word_set() { + local actual="$1" expected="$2" label="$3" actual_sorted expected_sorted word + actual_sorted="$(for word in $actual; do printf '%s\n' "${word#-}"; done | sort)" + expected_sorted="$(for word in $expected; do printf '%s\n' "${word#-}"; done | sort)" + if [ "$actual_sorted" != "$expected_sorted" ]; then + echo "Service effective $label differs from the reviewed exact allowlist." >&2 + return 1 + fi +} + assert_service_running() { - local active_state sub_state main_pid unit_user process_user + local require_reviewed="${1:-true}" + local active_state sub_state main_pid unit_user process_user teleo_uid teleo_gid + local uid_fields gid_fields groups_field exec_start process_executable + local seccomp_mode seccomp_filters lsm_context + local host_user_namespace service_user_namespace active_state="$(systemctl show "$SERVICE" --property=ActiveState --value)" sub_state="$(systemctl show "$SERVICE" --property=SubState --value)" if [ "$active_state" != "active" ] || [ "$sub_state" != "running" ]; then @@ -173,17 +215,91 @@ assert_service_running() { echo "Service process user is not teleo: $process_user" >&2 return 1 fi + if [ "$require_reviewed" = true ]; then + teleo_uid="$(id -u teleo)" + teleo_gid="$(id -g teleo)" + uid_fields="$(awk '/^Uid:/{print $2 ":" $3 ":" $4 ":" $5}' "/proc/$main_pid/status")" + gid_fields="$(awk '/^Gid:/{print $2 ":" $3 ":" $4 ":" $5}' "/proc/$main_pid/status")" + groups_field="$(awk '/^Groups:/{for (i=2;i<=NF;i++) printf "%s%s", (i==2 ? "" : ":"), $i}' "/proc/$main_pid/status")" + if [ "$uid_fields" != "$teleo_uid:$teleo_uid:$teleo_uid:$teleo_uid" ] || + [ "$gid_fields" != "$teleo_gid:$teleo_gid:$teleo_gid:$teleo_gid" ] || + [ "$groups_field" != "$teleo_gid" ]; then + echo "Service process does not have the exact reviewed uid/gid/group set." >&2 + return 1 + fi + fi + exec_start="$(systemctl show "$SERVICE" --property=ExecStart --value)" + case "$exec_start" in + *"path=$EXPECTED_HERMES_EXECUTABLE ; argv[]=$EXPECTED_HERMES_EXECUTABLE -p leoclean gateway run ;"*) ;; + *) + echo "Service effective ExecStart is not the reviewed leoclean Hermes gateway." >&2 + return 1 + ;; + esac + sudo /usr/bin/python3 -I - "$main_pid" "$EXPECTED_HERMES_PYTHON" "$EXPECTED_HERMES_EXECUTABLE" <<'PY' +import os +import sys +from pathlib import Path + +pid, expected_python, expected_hermes = sys.argv[1:] +raw = Path(f"/proc/{pid}/cmdline").read_bytes() +if not raw.endswith(b"\0") or b"\0\0" in raw: + raise SystemExit("service cmdline is malformed") +try: + argv = [part.decode("utf-8", "strict") for part in raw[:-1].split(b"\0")] +except UnicodeDecodeError: + raise SystemExit("service cmdline is not valid UTF-8") from None +if argv != [expected_python, expected_hermes, "-p", "leoclean", "gateway", "run"]: + raise SystemExit("service cmdline is not the reviewed leoclean Hermes gateway") +if os.path.realpath(f"/proc/{pid}/exe") != os.path.realpath(expected_python): + raise SystemExit("service executable is not the reviewed Hermes interpreter") +PY + if [ "$require_reviewed" = true ]; then + for capability_field in CapInh CapPrm CapEff CapBnd CapAmb; do + if [ "$(awk -v key="$capability_field:" '$1 == key {print $2}' "/proc/$main_pid/status")" != 0000000000000000 ]; then + echo "Service process retains capabilities in $capability_field." >&2 + return 1 + fi + done + if [ "$(awk '/^NoNewPrivs:/{print $2}' "/proc/$main_pid/status")" != 1 ]; then + echo "Service process does not enforce no-new-privileges." >&2 + return 1 + fi + seccomp_mode="$(awk '/^Seccomp:/{print $2}' "/proc/$main_pid/status")" + seccomp_filters="$(awk '/^Seccomp_filters:/{print $2}' "/proc/$main_pid/status")" + lsm_context="$(sudo cat "/proc/$main_pid/attr/current")" + host_user_namespace="$(sudo stat -Lc '%d:%i' /proc/1/ns/user)" + service_user_namespace="$(sudo stat -Lc '%d:%i' "/proc/$main_pid/ns/user")" + if [ "$seccomp_mode" != 0 ] || + { [ -n "$seccomp_filters" ] && [ "$seccomp_filters" != 0 ]; } || + [ "$lsm_context" != unconfined ] || + [ "$service_user_namespace" != "$host_user_namespace" ]; then + echo "Service process has an unreviewed live seccomp, LSM, or user-namespace restriction." >&2 + return 1 + fi + fi } assert_unit_configuration() { local require_reviewed="${1:-true}" local dropin_path dropin_name dropin_paths environment_files reviewed_dropin_seen=false + local property value capability_set ambient_capabilities no_new_privileges unit_group supplementary_groups + local inaccessible_paths credential_path + local read_only_paths read_write_paths bind_paths bind_read_only_paths temporary_file_systems + local private_network private_users network_property dropin_paths="$(systemctl show "$SERVICE" --property=DropInPaths --value)" environment_files="$(systemctl show "$SERVICE" --property=EnvironmentFiles --value)" if [ -n "$environment_files" ]; then echo "Service has an effective EnvironmentFile, which is not allowed for the scoped runtime." >&2 return 1 fi + for property in LoadCredential LoadCredentialEncrypted SetCredential SetCredentialEncrypted ImportCredential ExecCondition ExecStartPre ExecStartPost ExecReload ExecStop ExecStopPost; do + value="$(systemctl show "$SERVICE" --property="$property" --value)" + if [ -n "$value" ]; then + echo "Service has an effective $property credential source." >&2 + return 1 + fi + done for dropin_path in $dropin_paths; do if [ "$dropin_path" = "$REMOTE_DROPIN_DIR/$DROPIN_NAME" ]; then reviewed_dropin_seen=true @@ -199,13 +315,82 @@ assert_unit_configuration() { echo "Service does not load the reviewed Cloud SQL drop-in." >&2 return 1 fi + if [ "$require_reviewed" = true ]; then + capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)" + ambient_capabilities="$(systemctl show "$SERVICE" --property=AmbientCapabilities --value)" + no_new_privileges="$(systemctl show "$SERVICE" --property=NoNewPrivileges --value)" + unit_group="$(systemctl show "$SERVICE" --property=Group --value)" + supplementary_groups="$(systemctl show "$SERVICE" --property=SupplementaryGroups --value)" + inaccessible_paths="$(systemctl show "$SERVICE" --property=InaccessiblePaths --value)" + read_only_paths="$(systemctl show "$SERVICE" --property=ReadOnlyPaths --value)" + read_write_paths="$(systemctl show "$SERVICE" --property=ReadWritePaths --value)" + bind_paths="$(systemctl show "$SERVICE" --property=BindPaths --value)" + bind_read_only_paths="$(systemctl show "$SERVICE" --property=BindReadOnlyPaths --value)" + temporary_file_systems="$(systemctl show "$SERVICE" --property=TemporaryFileSystem --value)" + if [ -n "$capability_set" ] || [ -n "$ambient_capabilities" ] || [ "$no_new_privileges" != yes ] || + [ "$unit_group" != teleo ] || [ "$supplementary_groups" != teleo ]; then + echo "Service effective capability, privilege, or group contract differs from the reviewed drop-in." >&2 + return 1 + fi + assert_exact_word_set "$read_only_paths" "/home/teleo" ReadOnlyPaths + assert_exact_word_set "$read_write_paths" "$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" ReadWritePaths + assert_exact_word_set "$bind_paths" "" BindPaths + assert_exact_word_set "$bind_read_only_paths" "$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" BindReadOnlyPaths + assert_exact_word_set "$temporary_file_systems" "" TemporaryFileSystem + assert_exact_word_set "$supplementary_groups" "teleo" SupplementaryGroups + assert_exact_word_set "$inaccessible_paths" "/home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql" InaccessiblePaths + private_network="$(systemctl show "$SERVICE" --property=PrivateNetwork --value)" + private_users="$(systemctl show "$SERVICE" --property=PrivateUsers --value)" + if [ "$private_network" != no ] || [ "$private_users" != no ]; then + echo "Service effective private network or user namespace differs from the reviewed host contract." >&2 + return 1 + fi + for network_property in \ + NetworkNamespacePath \ + IPAddressDeny \ + IPAddressAllow \ + RestrictAddressFamilies \ + RestrictNetworkInterfaces \ + SocketBindAllow \ + SocketBindDeny \ + SystemCallFilter \ + AppArmorProfile \ + SELinuxContext; do + value="$(systemctl show "$SERVICE" --property="$network_property" --value)" + if [ -n "$value" ]; then + echo "Service effective $network_property differs from the reviewed unrestricted transport contract." >&2 + return 1 + fi + done + for credential_path in /home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql; do + case " $inaccessible_paths " in + *" $credential_path "*) ;; + *) + echo "Service does not mask a default credential path: $credential_path" >&2 + return 1 + ;; + esac + done + fi } assert_runtime_environment() { local verifier_path="$1" require_reviewed="${2:-true}" main_pid main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" - sudo /usr/bin/python3 "$verifier_path" --pid "$main_pid" + if [ "$require_reviewed" = true ]; then + sudo /usr/bin/python3 -I "$verifier_path" --pid "$main_pid" + else + sudo /usr/bin/python3 -I "$verifier_path" --pid "$main_pid" --allow-missing-runtime-security-fields + fi assert_unit_configuration "$require_reviewed" + if [ "$require_reviewed" = true ]; then + sudo nsenter -t "$main_pid" -m -- test ! -e "/run/credentials/$SERVICE" + for credential_path in /home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql; do + sudo nsenter -t "$main_pid" -m -- \ + /usr/bin/setpriv --reuid=teleo --regid=teleo --clear-groups --no-new-privs \ + /usr/bin/test ! -r "$credential_path" + done + fi } assert_attached_service_account() { @@ -225,17 +410,240 @@ assert_attached_service_account() { echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email" } +run_in_service_cgroup() { + local main_pid="$1" control_group + shift + control_group="$(systemctl show "$SERVICE" --property=ControlGroup --value)" + if [ -z "$control_group" ]; then + echo "Service has no effective cgroup." >&2 + return 1 + fi + sudo /usr/bin/python3 -I - "$main_pid" "$control_group" "$@" <<'PY' +import ctypes +import os +import resource +import signal +import stat +import sys +from pathlib import Path, PurePosixPath + + +HANDLED_SIGNALS = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP) +PR_SET_PDEATHSIG = 1 + + +class ParentInterrupted(BaseException): + def __init__(self, signum: int) -> None: + self.signum = signum + + +def fail() -> None: + raise SystemExit("service-context runner failed closed") + + +def process_cgroup(pid: str) -> str: + try: + lines = Path(f"/proc/{pid}/cgroup").read_text(encoding="ascii").splitlines() + except (OSError, UnicodeError): + fail() + matches = [] + for line in lines: + parts = line.split(":", 2) + if len(parts) == 3 and parts[0] == "0" and parts[1] == "": + matches.append(parts[2]) + if len(matches) != 1: + fail() + value = matches[0] + path = PurePosixPath(value) + if not value.startswith("/") or ".." in path.parts or str(path) != value: + fail() + return value + + +def set_parent_death_signal(expected_parent: int) -> None: + libc = ctypes.CDLL(None, use_errno=True) + prctl = libc.prctl + prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] + prctl.restype = ctypes.c_int + if prctl(PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) != 0: + os._exit(125) + if os.getppid() != expected_parent: + os._exit(125) + + +def kill_and_reap(child: int) -> None: + try: + os.kill(child, signal.SIGKILL) + except ProcessLookupError: + pass + while True: + try: + os.waitpid(child, 0) + return + except InterruptedError: + continue + except ChildProcessError: + return + + +def supported_resource_ids() -> tuple[int, ...]: + identifiers = set() + for name in dir(resource): + if not name.startswith("RLIMIT_") or name == "RLIMIT_NLIMITS": + continue + identifier = getattr(resource, name) + if not isinstance(identifier, int) or identifier < 0: + continue + try: + resource.prlimit(os.getpid(), identifier) + except (OSError, ValueError): + continue + identifiers.add(identifier) + return tuple(sorted(identifiers)) + + +def copy_process_limits(source_pid: int, child: int) -> None: + snapshot = { + identifier: resource.prlimit(source_pid, identifier) + for identifier in supported_resource_ids() + } + for identifier, limits in snapshot.items(): + resource.prlimit(child, identifier, limits) + if resource.prlimit(child, identifier) != limits: + fail() + if any(resource.prlimit(source_pid, identifier) != limits for identifier, limits in snapshot.items()): + fail() + + +def no_op_stopped_child(_child: int) -> None: + return None + + +def run_stopped_child( + command: list[str], + expected_cgroup: str, + *, + move_child, + read_child_cgroup, + prepare_child=set_parent_death_signal, + configure_stopped_child=no_op_stopped_child, +) -> int: + if not command or not command[0].startswith("/"): + fail() + original_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + parent_pid = os.getpid() + try: + child = os.fork() + except BaseException: + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + raise + if child == 0: + try: + for signum in HANDLED_SIGNALS: + signal.signal(signum, signal.SIG_DFL) + prepare_child(parent_pid) + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + if os.getppid() != parent_pid: + os._exit(125) + os.kill(os.getpid(), signal.SIGSTOP) + os.execv(command[0], command) + except BaseException: + os._exit(126) + + previous_handlers = {} + reaped = False + interrupted = None + status = None + + def interrupt(signum, _frame) -> None: + signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + raise ParentInterrupted(signum) + + try: + for signum in HANDLED_SIGNALS: + previous_handlers[signum] = signal.signal(signum, interrupt) + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + waited, status = os.waitpid(child, os.WUNTRACED) + if waited != child or not os.WIFSTOPPED(status) or os.WSTOPSIG(status) != signal.SIGSTOP: + fail() + move_child(child) + if read_child_cgroup(str(child)) != expected_cgroup: + fail() + configure_stopped_child(child) + os.kill(child, signal.SIGCONT) + waited, status = os.waitpid(child, 0) + if waited != child: + fail() + reaped = True + except ParentInterrupted as exc: + interrupted = exc.signum + finally: + signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + if not reaped: + kill_and_reap(child) + for signum, previous in previous_handlers.items(): + signal.signal(signum, previous) + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + + if interrupted is not None: + signal.signal(interrupted, signal.SIG_DFL) + os.kill(os.getpid(), interrupted) + fail() + if status is None: + fail() + if os.WIFEXITED(status): + return os.WEXITSTATUS(status) + if os.WIFSIGNALED(status): + return 128 + os.WTERMSIG(status) + fail() + + +def main() -> int: + if len(sys.argv) < 4: + fail() + main_pid, expected_cgroup, *command = sys.argv[1:] + if not main_pid.isascii() or not main_pid.isdecimal() or int(main_pid) <= 0: + fail() + if process_cgroup(main_pid) != expected_cgroup: + fail() + cgroup_root = Path("/sys/fs/cgroup").resolve(strict=True) + cgroup_dir = (cgroup_root / expected_cgroup.lstrip("/")).resolve(strict=True) + try: + cgroup_dir.relative_to(cgroup_root) + except ValueError: + fail() + cgroup_procs = cgroup_dir / "cgroup.procs" + try: + if not stat.S_ISREG(cgroup_procs.stat().st_mode): + fail() + except OSError: + fail() + + def move_child(child: int) -> None: + with cgroup_procs.open("w", encoding="ascii") as handle: + handle.write(f"{child}\n") + + return run_stopped_child( + command, + expected_cgroup, + move_child=move_child, + read_child_cgroup=process_cgroup, + configure_stopped_child=lambda child: copy_process_limits(int(main_pid), child), + ) + + +if __name__ == "__main__": + raise SystemExit(main()) +PY +} + assert_administrator_secret_denied() { - local main_pid + local main_pid probe_program main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" - sudo nsenter -t "$main_pid" -m --env -- \ - /usr/bin/setpriv \ - --reuid=teleo \ - --regid=teleo \ - --init-groups \ - --no-new-privs \ - /usr/bin/python3 - "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" <<'PY' + probe_program="$(cat <<'PY' import json +import os +import subprocess import sys import urllib.error import urllib.parse @@ -280,20 +688,69 @@ try: else: response.close() raise RuntimeError + gcloud_command = [ + "/usr/bin/gcloud", + "secrets", + "versions", + "access", + "latest", + "--secret=gcp-teleo-pgvector-standby-postgres-password", + f"--project={project}", + "--quiet", + ] + default_discovery_environment = os.environ.copy() + for key in ( + "CLOUDSDK_CONFIG", + "GOOGLE_APPLICATION_CREDENTIALS", + "PGPASSFILE", + "PGSERVICE", + "XDG_CONFIG_HOME", + ): + default_discovery_environment.pop(key, None) + default_discovery_environment["HOME"] = "/home/teleo" + for environment in (os.environ.copy(), default_discovery_environment): + denied = subprocess.run( + gcloud_command, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + env=environment, + timeout=30, + check=False, + ) + denial_error = denied.stderr[:65536] + if denied.returncode == 0: + raise RuntimeError + if "PERMISSION_DENIED" not in denial_error or "secretmanager.versions.access" not in denial_error: + raise RuntimeError except Exception: raise SystemExit("administrator secret was not the exact expected metadata-identity IAM denial") from None print(json.dumps({ "administrator_secret_access": "iam_permission_denied", + "ambient_gcloud_administrator_secret_access": "iam_permission_denied", "credential_context": "systemd_main_pid_environment", - "identity_source": "gce_metadata", + "direct_metadata_administrator_secret_access": "iam_permission_denied", + "default_discovery_administrator_secret_access": "iam_permission_denied", + "identity_source": "gce_metadata_ambient_gcloud_and_default_discovery", "stdout_discarded": True, }, sort_keys=True)) PY +)" + run_in_service_cgroup "$main_pid" \ + /usr/bin/nsenter -t "$main_pid" -m -n --env -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --clear-groups \ + --no-new-privs \ + /usr/bin/python3 -I -c "$probe_program" "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" } run_scoped_status() { - local phase="$1" tool_path="$2" + local phase="$1" tool_path="$2" server_ca_path="${3:-$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem}" + local cloudsdk_config="${4:-$REMOTE_RUNTIME_BIN/gcloud-config}" assert_attached_service_account echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical" sudo -n -u teleo env -i \ @@ -308,7 +765,12 @@ run_scoped_status() { TELEO_CLOUDSQL_USER=leoclean_kb_runtime \ TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \ TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \ - /usr/bin/python3 "$tool_path" status --format json --redacted + CLOUDSDK_CONFIG="$cloudsdk_config" \ + PGSSLMODE=verify-ca \ + PGSSLROOTCERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ + /usr/bin/python3 -I "$tool_path" status --format json --redacted } run_service_wrapper_status() { @@ -318,11 +780,12 @@ run_service_wrapper_status() { echo "SCOPED_WRAPPER_STATUS phase=$phase user=teleo database=teleo_canonical" # --env imports the target MainPID's effective /proc environment. setpriv # drops back to the service's Unix identity without sudo resetting that env. - sudo nsenter -t "$main_pid" -m --env -- \ + run_in_service_cgroup "$main_pid" \ + /usr/bin/nsenter -t "$main_pid" -m -n --env -- \ /usr/bin/setpriv \ --reuid=teleo \ --regid=teleo \ - --init-groups \ + --clear-groups \ --no-new-privs \ "$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted } @@ -346,22 +809,44 @@ assert_service_fingerprint() { } run_permission_verifier() { - local phase="$1" verifier_path="$2" receipt_path="$3" receipt_dir run_id receipt_state + local phase="$1" verifier_path="$2" receipt_path="$3" + local server_ca_path="${4:-$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem}" receipt_dir run_id receipt_state + local main_pid + local cloudsdk_config="${5:-$REMOTE_RUNTIME_BIN/gcloud-config}" run_id="$(date -u +%Y%m%d%H%M%S)-$phase" receipt_dir="$(dirname "$receipt_path")" sudo -n -u teleo mkdir -p -- "$receipt_dir" sudo -n -u teleo chmod 0700 "$receipt_dir" echo "NEGATIVE_PERMISSION_RECEIPT phase=$phase run_id=$run_id" - sudo -n -u teleo env -i \ - HOME=/home/teleo \ - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - /usr/bin/python3 "$verifier_path" --run-id "$run_id" --output "$receipt_path" + if [ "$phase" = pre ]; then + sudo -n -u teleo env -i \ + HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ + /usr/bin/python3 -I "$verifier_path" --run-id "$run_id" --output "$receipt_path" + else + main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" + run_in_service_cgroup "$main_pid" \ + /usr/bin/nsenter -t "$main_pid" -m -n -- \ + /usr/bin/setpriv \ + --reuid=teleo \ + --regid=teleo \ + --clear-groups \ + --no-new-privs \ + /usr/bin/env -i \ + HOME=/home/teleo \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ + TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ + /usr/bin/python3 -I "$verifier_path" --run-id "$run_id" --output "$receipt_path" + fi receipt_state="$(sudo stat -c '%U:%G:%a' "$receipt_path")" if [ "$receipt_state" != "teleo:teleo:600" ]; then echo "Permission receipt has unexpected ownership or mode: $receipt_state" >&2 return 1 fi - sudo /usr/bin/python3 - "$receipt_path" <<'PY' + sudo /usr/bin/python3 -I - "$receipt_path" <<'PY' import json import sys from pathlib import Path @@ -381,33 +866,74 @@ print(json.dumps({"permission_receipt": "validated", "validated_fields": sorted( PY } +assert_trusted_executable() { + local requested="$1" resolved current owner + sudo test -x "$requested" + owner="$(sudo stat -c '%U' "$requested")" + if [ "$owner" != root ]; then + echo "Runtime executable path is not root-owned: $requested" >&2 + return 1 + fi + resolved="$(sudo readlink -f -- "$requested")" + sudo test -f "$resolved" + sudo test -x "$resolved" + current="$(dirname "$requested")" + while :; do + test "$(sudo stat -Lc '%U' "$current")" = root + sudo -n -u teleo test ! -w "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done + current="$resolved" + while :; do + test "$(sudo stat -Lc '%U' "$current")" = root + sudo -n -u teleo test ! -w "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done +} + assert_installed_files() { - local wrapper tool permission_verifier service_env_verifier dropin revision executable_dir protected_dir protected_file + local wrapper tool permission_verifier service_env_verifier server_ca gcloud_config dropin revision + local executable_dir protected_dir protected_file actual_entries expected_entries wrapper="$REMOTE_RUNTIME_BIN/teleo-kb" tool="$REMOTE_RUNTIME_BIN/cloudsql_memory_tool.py" permission_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" service_env_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" + server_ca="$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem" + gcloud_config="$REMOTE_RUNTIME_BIN/gcloud-config" dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME" revision="$REMOTE_RUNTIME_BIN/.livingip-runtime-revision" test "$(sudo sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA" test "$(sudo sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA" test "$(sudo sha256sum "$permission_verifier" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" test "$(sudo sha256sum "$service_env_verifier" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" + test "$(sudo sha256sum "$server_ca" | awk '{print $1}')" = "$SERVER_CA_SHA" test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA" test "$(sudo stat -c '%U:%G:%a' "$REMOTE_RUNTIME_BIN")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$wrapper")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$tool")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$permission_verifier")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$service_env_verifier")" = "root:root:755" + test "$(sudo stat -c '%U:%G:%a' "$server_ca")" = "root:root:644" + test "$(sudo stat -c '%U:%G:%a' "$gcloud_config")" = "root:root:555" test "$(sudo stat -c '%U:%G:%a' "$revision")" = "root:root:644" test "$(sudo stat -c '%U:%G:%a' "$dropin")" = "root:root:644" test "$(sudo cat "$revision")" = "$SOURCE_REVISION" + sudo test -z "$(sudo find "$gcloud_config" -mindepth 1 -print -quit)" + expected_entries="$(printf '%s\n' .livingip-runtime-revision cloudsql-server-ca.pem cloudsql_memory_tool.py gcloud-config teleo-kb verify_gcp_leoclean_runtime_permissions.py verify_gcp_leoclean_service_environment.py | sort)" + actual_entries="$(sudo find "$REMOTE_RUNTIME_BIN" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" + if [ "$actual_entries" != "$expected_entries" ]; then + echo "Runtime directory contains an unreviewed or missing entry." >&2 + return 1 + fi for protected_dir in \ /usr \ /usr/local \ /usr/local/libexec \ /usr/local/libexec/livingip \ "$REMOTE_RUNTIME_BIN" \ + "$gcloud_config" \ "$REMOTE_DROPIN_DIR"; do sudo test -d "$protected_dir" sudo test ! -L "$protected_dir" @@ -419,6 +945,7 @@ assert_installed_files() { "$tool" \ "$permission_verifier" \ "$service_env_verifier" \ + "$server_ca" \ "$revision" \ "$dropin"; do sudo test -f "$protected_file" @@ -436,22 +963,27 @@ assert_installed_files() { test "$(sudo stat -Lc '%U:%G' "$executable_dir")" = "root:root" sudo -n -u teleo test ! -w "$executable_dir" done - for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3; do - sudo test -x "$protected_file" - test "$(sudo stat -Lc '%U:%G' "$protected_file")" = "root:root" - sudo -n -u teleo test ! -w "$protected_file" + for protected_file in /bin/bash /usr/bin/curl /usr/bin/env /usr/bin/gcloud /usr/bin/psql /usr/bin/python3 /usr/bin/setpriv; do + assert_trusted_executable "$protected_file" done sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin" sudo grep -E '^Environment=TELEO_(KB_MODE|GCP_PROJECT|GCP_METADATA_ONLY|CLOUDSQL_|CANONICAL_CLOUDSQL_DB)' "$dropin" - sudo grep -Fx "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$dropin" + sudo grep -Fx 'ReadOnlyPaths=/home/teleo' "$dropin" sudo grep -Fx "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$dropin" sudo grep -Fx "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$dropin" - sudo grep -Fx 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$dropin" + sudo grep -Fx 'CapabilityBoundingSet=' "$dropin" + sudo grep -Fx 'AmbientCapabilities=' "$dropin" + sudo grep -Fx 'NoNewPrivileges=yes' "$dropin" + sudo grep -Fx 'Group=teleo' "$dropin" + sudo grep -Fx 'SupplementaryGroups=teleo' "$dropin" + sudo grep -Fx \ + 'UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT' \ + "$dropin" } assert_deployed_files() { local main_pid namespace_state source_state mount_state - local hermes_mount_state state_mount_state workspace_mount_state capability_set + local home_mount_state state_mount_state workspace_mount_state capability_set ambient_capabilities assert_installed_files main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" @@ -469,13 +1001,13 @@ assert_deployed_files() { return 1 ;; esac - hermes_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_HERMES_ROOT" -n -o OPTIONS)" + home_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M /home/teleo -n -o OPTIONS)" state_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/state" -n -o OPTIONS)" workspace_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/workspace" -n -o OPTIONS)" - case ",$hermes_mount_state," in + case ",$home_mount_state," in *,ro,*) ;; *) - echo "Service .hermes ancestor mount is not read-only." >&2 + echo "Service home mount is not read-only." >&2 return 1 ;; esac @@ -501,37 +1033,42 @@ assert_deployed_files() { sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/state" sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/workspace" capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)" - if printf '%s\n' "$capability_set" | grep -Eqi '(^|[[:space:]])cap_sys_admin([[:space:]]|$)'; then - echo "Service capability bounding set still contains CAP_SYS_ADMIN." >&2 + ambient_capabilities="$(systemctl show "$SERVICE" --property=AmbientCapabilities --value)" + if [ -n "$capability_set" ] || [ -n "$ambient_capabilities" ]; then + echo "Service effective capability sets are not empty." >&2 return 1 fi test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb" | awk '{print $1}')" = "$WRAPPER_SHA" test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql_memory_tool.py" | awk '{print $1}')" = "$TOOL_SHA" test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_runtime_permissions.py" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_service_environment.py" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" + test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql-server-ca.pem" | awk '{print $1}')" = "$SERVER_CA_SHA" } REMOTE ) printf -v verify_context \ - 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \ - "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nEXPECTED_HERMES_EXECUTABLE=%q\nEXPECTED_HERMES_PYTHON=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nSERVER_CA_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$EXPECTED_HERMES_EXECUTABLE" "$EXPECTED_HERMES_PYTHON" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ - "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" "$SOURCE_REVISION" + "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$SERVER_CA_SHA" "$DROPIN_SHA" "$SOURCE_REVISION" verify_body=$(cat <<'REMOTE' set -euo pipefail -verification_tmp="$(mktemp -d /tmp/teleo-gcp-runtime-verify.XXXXXX)" +verification_tmp="$(sudo mktemp -d /run/teleo-gcp-runtime-verify.XXXXXX)" sudo chown teleo:teleo "$verification_tmp" sudo chmod 0700 "$verification_tmp" cleanup_verification() { sudo rm -rf -- "$verification_tmp"; } trap cleanup_verification EXIT assert_deployed_files assert_service_running +verification_service_before="$(service_fingerprint)" assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" run_service_wrapper_status verify assert_administrator_secret_denied run_permission_verifier verify "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" "$verification_tmp/permission-receipt.json" +assert_service_fingerprint "$verification_service_before" +assert_service_running systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager REMOTE ) @@ -539,7 +1076,7 @@ verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}" if $DRY_RUN; then echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE preflight_only=$PREFLIGHT_ONLY restart=$RESTART" - echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA dropin_sha256=$DROPIN_SHA" + echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA server_ca_sha256=$SERVER_CA_SHA dropin_sha256=$DROPIN_SHA" exit 0 fi @@ -554,15 +1091,15 @@ if ! $RESTART && ! $PREFLIGHT_ONLY; then fi printf -v sync_context \ - 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \ - "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ + 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nEXPECTED_HERMES_EXECUTABLE=%q\nEXPECTED_HERMES_PYTHON=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nSERVER_CA_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nSERVER_CA_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \ + "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$EXPECTED_HERMES_EXECUTABLE" "$EXPECTED_HERMES_PYTHON" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ - "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" \ - "$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY" + "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$SERVER_CA_SHA" "$DROPIN_SHA" \ + "$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$SERVER_CA_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY" sync_body=$(cat <<'REMOTE' set -euo pipefail -remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)" +remote_tmp="$(sudo mktemp -d /run/teleo-gcp-leoclean-runtime.XXXXXX)" runtime_bin="$REMOTE_RUNTIME_BIN" runtime_parent="$(dirname "$runtime_bin")" runtime_grandparent="$(dirname "$runtime_parent")" @@ -576,6 +1113,7 @@ wrapper_stage="$runtime_bin/.teleo-kb.livingip-new.$$" tool_stage="$runtime_bin/.cloudsql-memory-tool.livingip-new.$$" permission_verifier_stage="$runtime_bin/.runtime-permission-verifier.livingip-new.$$" service_env_verifier_stage="$runtime_bin/.service-environment-verifier.livingip-new.$$" +server_ca_stage="$runtime_bin/.cloudsql-server-ca.livingip-new.$$" revision_stage="$runtime_bin/.livingip-runtime-revision.livingip-new.$$" dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$" mutation_started=false @@ -679,8 +1217,10 @@ rollback_runtime() { restore_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 restore_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 restore_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 + restore_path "$runtime_bin/cloudsql-server-ca.pem" server_ca || rollback_rc=1 restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 restore_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 + restore_directory_state "$runtime_bin/gcloud-config" gcloud_config || rollback_rc=1 restore_directory_state "$runtime_bin" runtime_bin || rollback_rc=1 restore_directory_state "$runtime_parent" runtime_parent || rollback_rc=1 restore_directory_state "$runtime_grandparent" runtime_grandparent || rollback_rc=1 @@ -690,11 +1230,13 @@ rollback_runtime() { validate_restored_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 validate_restored_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 validate_restored_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 + validate_restored_path "$runtime_bin/cloudsql-server-ca.pem" server_ca || rollback_rc=1 validate_restored_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 validate_restored_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 validate_restored_directory "$runtime_grandparent" runtime_grandparent || rollback_rc=1 validate_restored_directory "$runtime_parent" runtime_parent || rollback_rc=1 validate_restored_directory "$runtime_bin" runtime_bin || rollback_rc=1 + validate_restored_directory "$runtime_bin/gcloud-config" gcloud_config || rollback_rc=1 validate_restored_directory "$dropin_dir" dropin_dir || rollback_rc=1 fi if [ "$rollback_rc" -eq 0 ]; then @@ -705,7 +1247,7 @@ rollback_runtime() { fi if [ "$rollback_rc" -eq 0 ]; then sudo systemctl start "$SERVICE" || rollback_rc=1 - assert_service_running || rollback_rc=1 + assert_service_running false || rollback_rc=1 fi return "$rollback_rc" } @@ -728,6 +1270,7 @@ on_exit() { "$tool_stage" \ "$permission_verifier_stage" \ "$service_env_verifier_stage" \ + "$server_ca_stage" \ "$revision_stage" \ "$dropin_stage" || true sudo rm -rf -- "$remote_tmp" || true @@ -754,11 +1297,13 @@ fi sudo chown -R root:root "$payload_dir" sudo find "$payload_dir" -type d -exec chmod 0555 {} + sudo find "$payload_dir" -type f -exec chmod 0444 {} + +sudo install -d -o root -g root -m 0555 "$payload_dir/gcloud-config" sudo chmod 0555 \ "$payload_dir/$WRAPPER_REL" \ "$payload_dir/$TOOL_REL" \ "$payload_dir/$PERMISSION_VERIFIER_REL" \ "$payload_dir/$SERVICE_ENV_VERIFIER_REL" +test "$(sudo sha256sum "$payload_dir/$SERVER_CA_REL" | awk '{print $1}')" = "$SERVER_CA_SHA" test "$(sudo sha256sum "$payload_dir/$WRAPPER_REL" | awk '{print $1}')" = "$WRAPPER_SHA" test "$(sudo sha256sum "$payload_dir/$TOOL_REL" | awk '{print $1}')" = "$TOOL_SHA" test "$(sudo sha256sum "$payload_dir/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" @@ -783,6 +1328,7 @@ for existing_directory in \ "$runtime_grandparent" \ "$runtime_parent" \ "$runtime_bin" \ + "$runtime_bin/gcloud-config" \ "$dropin_dir"; do assert_existing_root_directory "$existing_directory" done @@ -791,19 +1337,24 @@ for existing_file in \ "$runtime_bin/cloudsql_memory_tool.py" \ "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" \ "$runtime_bin/verify_gcp_leoclean_service_environment.py" \ + "$runtime_bin/cloudsql-server-ca.pem" \ "$runtime_bin/.livingip-runtime-revision" \ "$dropin_dir/$DROPIN_NAME"; do assert_existing_root_file "$existing_file" done +if sudo test -d "$runtime_bin/gcloud-config"; then + sudo test -z "$(sudo find "$runtime_bin/gcloud-config" -mindepth 1 -print -quit)" +fi -assert_service_running -preflight_service_before="$(service_fingerprint)" -assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false -run_scoped_status pre "$payload_dir/$TOOL_REL" -assert_administrator_secret_denied -run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" -assert_service_fingerprint "$preflight_service_before" +assert_service_running false if [ "$PREFLIGHT_ONLY" = true ]; then + preflight_service_before="$(service_fingerprint)" + assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false + run_scoped_status pre "$payload_dir/$TOOL_REL" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" + assert_administrator_secret_denied + run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" + assert_service_fingerprint "$preflight_service_before" + assert_service_running false systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager exit 0 fi @@ -812,17 +1363,29 @@ backup_path "$runtime_bin/teleo-kb" wrapper backup_path "$runtime_bin/cloudsql_memory_tool.py" tool backup_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier backup_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier +backup_path "$runtime_bin/cloudsql-server-ca.pem" server_ca backup_path "$dropin_dir/$DROPIN_NAME" dropin backup_path "$runtime_bin/.livingip-runtime-revision" revision backup_directory_state "$runtime_parent" runtime_parent backup_directory_state "$runtime_grandparent" runtime_grandparent backup_directory_state "$runtime_bin" runtime_bin +backup_directory_state "$runtime_bin/gcloud-config" gcloud_config backup_directory_state "$dropin_dir" dropin_dir +preflight_service_before="$(service_fingerprint)" +assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false +run_scoped_status pre "$payload_dir/$TOOL_REL" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" +assert_administrator_secret_denied +run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" +assert_service_fingerprint "$preflight_service_before" +assert_service_running false mutation_started=true sudo systemctl stop "$SERVICE" sudo install -d -o root -g root -m 0755 "$runtime_parent" sudo install -d -o root -g root -m 0755 "$runtime_bin" +sudo install -d -o root -g root -m 0555 "$runtime_bin/gcloud-config" +sudo install -o root -g root -m 0644 "$payload_dir/$SERVER_CA_REL" "$server_ca_stage" +sudo mv -f -- "$server_ca_stage" "$runtime_bin/cloudsql-server-ca.pem" sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage" sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage" sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" "$permission_verifier_stage" @@ -845,10 +1408,13 @@ sudo systemctl start "$SERVICE" assert_deployed_files assert_service_running +post_service_before="$(service_fingerprint)" assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py" run_service_wrapper_status post assert_administrator_secret_denied run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" "$work_dir/permission-post/receipt.json" +assert_service_fingerprint "$post_service_before" +assert_service_running systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager mutation_started=false REMOTE @@ -860,4 +1426,5 @@ COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ + "$SERVER_CA_REL" \ "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command" diff --git a/docs/gcp-leoclean-runtime-reconciliation.md b/docs/gcp-leoclean-runtime-reconciliation.md index eafcb50..3ac69de 100644 --- a/docs/gcp-leoclean-runtime-reconciliation.md +++ b/docs/gcp-leoclean-runtime-reconciliation.md @@ -44,6 +44,11 @@ IAM or networking. 7. Deployment proves the attached VM service account from the metadata server, uses an empty phase-local Cloud SDK configuration, and rolls the complete prior runtime set back if installation or post-restart verification fails. +8. Post-restart status, permission, and administrator-secret denial probes join + both the actual service mount and network namespaces. Effective systemd + network, address-family, socket-bind, syscall-filter, and LSM properties must + match the reviewed unrestricted transport contract so a host-context helper + cannot produce a false runtime receipt. ## Safe Order Of Operations @@ -70,21 +75,33 @@ an artifact, or a command transcript. ### 4. Provision the scoped PostgreSQL role once -Run `ops/gcp_leoclean_runtime_role.sql` as the Cloud SQL administrator from the -private VM path. Supply the new runtime password through -`TELEO_LEOCLEAN_DB_PASSWORD`; the SQL file does not contain it. The migration: +Run `ops/provision_gcp_leoclean_runtime_role.sh` as the Cloud SQL administrator +from the private VM path. Supply the new runtime password through +`TELEO_LEOCLEAN_DB_PASSWORD` and the administrator password through +`PGPASSWORD`. The wrapper captures and unsets both fields before its first +external child, pins the private endpoint and reviewed server CA, detaches +`psql` from the controlling terminal, and feeds the runtime value only to +psql's client-side `\password` prompt. The SQL file and argv never contain the +cleartext runtime password. The migration: - creates or rotates `leoclean_kb_runtime`; -- keeps `leoclean_kb_runtime` `NOLOGIN` while both the `teleo_canonical` grant - phase and the legacy `teleo_kb` denial-verification phase run, and enables - `LOGIN` only in the final transaction after both phases pass; +- changes any pre-existing `leoclean_kb_runtime` to `NOLOGIN`, removes its + membership edges, and terminates its existing sessions before password + handling; an interruption therefore leaves the old login disabled; +- enables `LOGIN` only as the last statement in the same transaction as the + database ACL, canonical grant, routine, large-object, and topology checks; - creates a dedicated `NOLOGIN` function owner with no role memberships; - removes stale table, column, sequence, function, and role-membership grants; +- inventories existing principals, replaces `PUBLIC CONNECT` with explicit + preserved grants, gives the runtime `CONNECT` only to `teleo_canonical`, and + leaves the staging owner with no database connection target; +- inventories and preserves existing non-scoped application-routine and + large-object access while removing `PUBLIC` and both scoped roles from all + other application routines and every persistent large-object mutator; - grants exact canonical reads; - creates a locked security-definer staging function that hard-codes both `pending_review` and canonical proposer `leo`; - grants no direct table writes; -- removes all access to the legacy `teleo_restore` schema; - aborts if either scoped role owns or can reach anything outside the explicit allowlist. @@ -99,11 +116,13 @@ by a `pg_catalog, pg_temp` search path, schema-qualified relations, a fixed The administrator password is used only for this bounded bootstrap. Retain no password output. -`teleo_kb` can still be connectable through its existing `PUBLIC CONNECT` -grant. That database-wide default is not treated as runtime authority. The -migration denies the scoped role access to the `teleo_restore` schema, the -runtime client is pinned to `teleo_canonical`, and audit fallback remains -disabled with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`. +`teleo_kb`, `template1`, and every other noncanonical database are actual +negative connection checks. PostgreSQL's startup protocol does not expose a +SQLSTATE through `psql`, so the verifier requires the exact C-locale +`permission denied for database` and `does not have CONNECT privilege` +diagnostics in addition to the catalog proof that the runtime's sole effective +target is `teleo_canonical`. Audit fallback remains disabled with +`TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`. ### 5. Preflight and deploy the merged runtime @@ -143,41 +162,71 @@ directory as a read-only bind at its existing `/home/teleo/.hermes/profiles/leoclean/bin` path; verification compares the source and service-namespace inode and hashes and proves the mount is read-only. To prevent the `teleo` service user from renaming a writable ancestor and -recreating that absolute path, the service namespace mounts -`/home/teleo/.hermes` read-only, allows writes back only to the profile's -`state` and `workspace` subdirectories, and removes `CAP_SYS_ADMIN`. Live -verification requires `.hermes` and `bin` to be read-only, both allowed -subdirectories to be writable, `/home` to be non-writable by `teleo`, and the -effective capability set to exclude `CAP_SYS_ADMIN`. +recreating that absolute path, the service namespace mounts the complete +`/home/teleo` tree read-only, allows writes back only to the profile's `state` +and `workspace` subdirectories, clears every bounding and ambient capability, +and enforces `NoNewPrivileges`. Live verification requires the service home and +bound `bin` to be read-only, both allowed subdirectories to be writable, the +exact `teleo` group set, and every process capability field to be zero. Preflight, post-restart, and `--verify-only` checks inspect the environment of the actual systemd `MainPID` through `/proc//environ`, rather than trusting the -configured unit environment alone. A standalone fail-closed verifier requires +configured unit environment alone. It also rejects effective credential +properties, auxiliary `Exec*` hooks, any `EnvironmentFile`, and any drop-in +ordered after the reviewed configuration. A standalone fail-closed verifier requires the exact reviewed private target and runtime role, requires `TELEO_GCP_METADATA_ONLY=1`, and rejects all PostgreSQL, Cloud SDK, proxy, TLS trust, broader Google credential, shell-startup, dynamic-loader, Python-loader, -and administrator-secret overrides without printing their values. Effective -`EnvironmentFile` use and drop-ins ordered after the reviewed configuration are -rejected before restart. This blocks +administrator-secret, excerpt, credential-mode, claim-base, and retry-count +overrides without printing their values. The reviewed root-owned empty +`CLOUDSDK_CONFIG`, pinned `PGSSLMODE=verify-ca`, +server CA, and Python isolation fields must match exactly. This blocks pre-wrapper injection through fields such as `BASH_ENV`, any `LD_*`, `PYTHONPATH`, `PYTHONHTTPSVERIFY`, or `SSLKEYLOGFILE`. The checks also require the reviewed drop-in to appear in `DropInPaths`. The wrapper status probe then imports the validated environment from the actual `MainPID` with `nsenter ---env`, enters its mount namespace, drops to `teleo`, and invokes the bound +--env`, enters its mount and network namespaces, drops to `teleo`, and invokes the bound wrapper without reinjecting any database identity setting. +The effective unit must use the host network namespace (`PrivateNetwork=no`), +have no `NetworkNamespacePath`, `IPAddressDeny`, `IPAddressAllow`, +`RestrictAddressFamilies`, `SocketBindAllow`, `SocketBindDeny`, +`RestrictNetworkInterfaces`, `SystemCallFilter`, `AppArmorProfile`, or +`SELinuxContext` override, and load no +later drop-in. The executable regression harness injects each conflicting +property and requires the gate to fail. These checks cover restrictions that a +new `nsenter` process would not inherit merely by sharing the service network +namespace. + +For cgroup/eBPF restrictions, post-restart probes do not rely on enumeration. +A root-only bounded runner verifies the service's single cgroup-v2 membership +against systemd's effective `ControlGroup`, forks a stopped child, moves only +that child into the service cgroup, confirms the move, and then resumes it to +join the service namespaces and drop to `teleo`. It propagates exit or signal +status and reaps the child on every path. The child installs a Linux +parent-death `SIGKILL` before stopping, with a parent-PID race check; the parent +blocks and handles `INT`, `TERM`, and `HUP` through cleanup before re-raising +the signal. Before resume, every supported Linux `RLIMIT_*` soft/hard pair is +copied from the live `MainPID` with a stable source and child readback. Live +`MainPID` verification also requires seccomp mode and filter count zero, the +exact `unconfined` LSM context, `PrivateUsers=no`, and the same user-namespace +inode as PID 1, preventing a per-process filter, profile, resource ceiling, or +user namespace from being mistaken for the reviewed service context. + The service runtime bypasses Cloud SDK configuration and credential caches entirely. The helper obtains the attached service-account email and a bounded access token from the fixed GCE metadata endpoint, requires the exact expected service account, and accesses only the named scoped Secret Manager version over the fixed HTTPS API. Its URL opener disables proxies. The scoped status read and administrator-secret IAM denial are rerun after restart. The denial probe -imports the actual systemd `MainPID` environment and mount namespace, drops to +imports the actual systemd `MainPID` environment and mount and network +namespaces, drops to the service's `teleo` identity, and must observe the exact `secretmanager.versions.access` permission denial. No token or -secret value is retained. The status read executes the bound `teleo-kb` wrapper -from inside the service mount namespace, proving the wrapper and its adjacent -helper rather than bypassing them with a direct helper invocation. A failure +secret value is retained. The status read and permission verifier execute from +inside the service mount and network namespaces; the status path invokes the +bound `teleo-kb` wrapper and its adjacent helper rather than bypassing them with +a direct helper invocation. A failure after mutation restores the prior runtime/drop-in directory existence and metadata, wrapper, helper, both verifiers, drop-in, and revision as one set, validates it against the root-only backup, and only then reloads systemd and @@ -196,7 +245,7 @@ sudo chmod 0700 "$receipt_dir" run_id="$(date -u +%Y%m%d%H%M%S)-manual" sudo -n -u teleo env -i HOME=/home/teleo \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - python3 /usr/local/libexec/livingip/leoclean-kb/verify_gcp_leoclean_runtime_permissions.py \ + /usr/bin/python3 -I /usr/local/libexec/livingip/leoclean-kb/verify_gcp_leoclean_runtime_permissions.py \ --run-id "$run_id" \ --output "$receipt_dir/permission-receipt.json" sudo stat -c '%U:%G:%a' "$receipt_dir/permission-receipt.json" @@ -230,13 +279,17 @@ Retain a redacted receipt containing: both before and after the transaction; - denied direct insert, update, and delete on `kb_stage.kb_proposals`; - denied insert, update, and delete on canonical `public.claims`; +- denied `lo_creat`, `lo_create`, and `lo_from_bytea`, zero effective + large-object mutator execution, and zero scoped large-object ownership/ACLs; - denied reviewer/apply security-definer functions; - exact function-catalog posture: only the five-argument staging function is executable, the forged six-argument overload is absent, and each expected reviewer/apply function exists but is not executable by the runtime role; - unavailable forged-proposer overload and denied `SET ROLE` escalation into the staging owner, reviewer, apply, or administrator roles; -- denied access to `teleo_restore` even when `teleo_kb` remains connectable; +- denied startup connections to both `teleo_kb` and `template1`, with the + catalog showing `teleo_canonical` as the sole runtime connection target and + zero remaining `PUBLIC CONNECT` grants; - readable scoped runtime secret and an IAM permission denial for the administrator secret, with neither secret value retained; - zero Telegram messages and zero committed canary rows. diff --git a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py index 3367207..9c579fd 100755 --- a/hermes-agent/leoclean-bin/cloudsql_memory_tool.py +++ b/hermes-agent/leoclean-bin/cloudsql_memory_tool.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 -I """Cloud SQL memory and KB-proposal bridge for the GCP leoclean profile. Canonical claim/strategy application remains review-gated. The production-like @@ -17,8 +17,10 @@ import hashlib import json import os import re +import stat import subprocess import urllib.request +from pathlib import Path from typing import Any STOPWORDS = { @@ -84,6 +86,12 @@ RUNTIME_GCP_PROJECT = "teleo-501523" RUNTIME_CLOUDSQL_HOST = "10.61.0.3" RUNTIME_CLOUDSQL_PORT = "5432" RUNTIME_CLOUDSQL_DB = "teleo_canonical" +RUNTIME_SYSTEM_IDENTIFIER = "7659718422914359312" +RUNTIME_CLOUDSDK_CONFIG = "/usr/local/libexec/livingip/leoclean-kb/gcloud-config" +RUNTIME_PSQL_BIN = "/usr/bin/psql" +RUNTIME_SSL_MODE = "verify-ca" +RUNTIME_SSL_ROOT_CERT = "/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem" +RUNTIME_SSL_ROOT_CERT_SHA256 = "80701e768f0e1f6b9d621aa0b53f6e851daaa276c6d9a8e51a300fbc015539cb" RUNTIME_SERVICE_ACCOUNT = "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" GCE_METADATA_EMAIL_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" GCE_METADATA_TOKEN_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" @@ -105,13 +113,60 @@ RUNTIME_TLS_TRUST_ENVIRONMENT = frozenset( "SSL_CERT_DIR", "SSL_CERT_FILE", "SSLKEYLOGFILE", + "OPENSSL_CONF", + "OPENSSL_ENGINES", + "OPENSSL_MODULES", + "GCONV_PATH", } ) RUNTIME_PSQL_ENV_BASE = { "PATH": "/usr/bin:/bin", - "PGSSLMODE": "require", + "PGSSLMODE": RUNTIME_SSL_MODE, "PGCONNECT_TIMEOUT": "8", } + + +def runtime_ssl_root_cert() -> str: + """Return the reviewed installed or ephemeral preflight CA path.""" + + raw = os.environ.get("TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT", RUNTIME_SSL_ROOT_CERT) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + if resolved != candidate or not resolved.is_file(): + raise OSError + for part in (resolved, *resolved.parents): + metadata = part.stat() + if metadata.st_uid != 0 or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise OSError + certificate = resolved.read_bytes() + except OSError: + raise SystemExit("Reviewed Cloud SQL server CA path is unavailable or untrusted.") from None + if hashlib.sha256(certificate).hexdigest() != RUNTIME_SSL_ROOT_CERT_SHA256: + raise SystemExit("Reviewed Cloud SQL server CA does not match the pinned certificate.") + return str(resolved) + + +def runtime_cloudsdk_config() -> str: + """Return a root-owned empty SDK configuration directory.""" + + raw = os.environ.get("TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG", RUNTIME_CLOUDSDK_CONFIG) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + if resolved != candidate or not resolved.is_dir() or any(resolved.iterdir()): + raise OSError + for part in (resolved, *resolved.parents): + metadata = part.stat() + if metadata.st_uid != 0 or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise OSError + except OSError: + raise SystemExit("Reviewed empty Cloud SDK configuration is unavailable or untrusted.") from None + return str(resolved) CLONE_READ_ONLY_PGOPTIONS = "-c default_transaction_read_only=on" RECEIPTED_READ_COMMANDS = frozenset( { @@ -290,6 +345,8 @@ def validate_runtime_connection(args: argparse.Namespace) -> None: password_secret = str(args.password_secret or "") mode = str(getattr(args, "credential_mode", "runtime")) if mode == "runtime": + ssl_root_cert = runtime_ssl_root_cert() + cloudsdk_config = runtime_cloudsdk_config() if user != RUNTIME_DB_USER: raise SystemExit(f"Leo runtime requires database user {RUNTIME_DB_USER}.") if password_secret != RUNTIME_PASSWORD_SECRET: @@ -310,12 +367,31 @@ def validate_runtime_connection(args: argparse.Namespace) -> None: if os.environ.get("TELEO_GCP_METADATA_ONLY") != "1": raise SystemExit("Leo runtime requires TELEO_GCP_METADATA_ONLY=1.") forbidden_environment = any( - normalized_key.startswith("PG") - or normalized_key.startswith("CLOUDSDK_") + ( + normalized_key.startswith("PG") + and not ( + (normalized_key == "PGSSLMODE" and value == RUNTIME_SSL_MODE) + or (normalized_key == "PGSSLROOTCERT" and value == ssl_root_cert) + ) + ) + or ( + normalized_key.startswith("CLOUDSDK_") + and not ( + normalized_key == "CLOUDSDK_CONFIG" + and value == cloudsdk_config + ) + ) or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS" or normalized_key.startswith("LD_") or normalized_key.startswith("BASH_FUNC_") - or (normalized_key.startswith("PYTHON") and normalized_key != "PYTHONUNBUFFERED") + or ( + normalized_key.startswith("PYTHON") + and not ( + (normalized_key == "PYTHONNOUSERSITE" and value == "1") + or (normalized_key == "PYTHONSAFEPATH" and value == "1") + or normalized_key == "PYTHONUNBUFFERED" + ) + ) or normalized_key in { "BASH_ENV", "BASHOPTS", @@ -329,7 +405,8 @@ def validate_runtime_connection(args: argparse.Namespace) -> None: or normalized_key == "PROXY" or normalized_key.endswith("_PROXY") or normalized_key in RUNTIME_TLS_TRUST_ENVIRONMENT - for normalized_key in (key.upper() for key in os.environ) + for key, value in os.environ.items() + for normalized_key in (key.upper(),) ) if forbidden_environment: raise SystemExit( @@ -515,6 +592,7 @@ def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: credential = password(args) if args.credential_mode == "runtime": env = dict(RUNTIME_PSQL_ENV_BASE) + env["PGSSLROOTCERT"] = runtime_ssl_root_cert() else: env = {key: value for key, value in os.environ.items() if not key.startswith("PG")} env.update( @@ -529,7 +607,7 @@ def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str: if args.credential_mode == "clone-readonly": env["PGOPTIONS"] = CLONE_READ_ONLY_PGOPTIONS result = subprocess.run( - ["psql", "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"], + [RUNTIME_PSQL_BIN, "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"], text=True, capture_output=True, input=sql, @@ -583,6 +661,8 @@ select jsonb_build_object( raise SystemExit("Canonical database read marker returned an unexpected server port.") if marker.get("ssl") is not True: raise SystemExit("Canonical database read marker did not prove an encrypted connection.") + if marker.get("system_identifier") != RUNTIME_SYSTEM_IDENTIFIER: + raise SystemExit("Canonical database read marker returned an unexpected system identifier.") return marker diff --git a/hermes-agent/leoclean-bin/teleo-kb-gcp b/hermes-agent/leoclean-bin/teleo-kb-gcp new file mode 100755 index 0000000..9e56c18 --- /dev/null +++ b/hermes-agent/leoclean-bin/teleo-kb-gcp @@ -0,0 +1,52 @@ +#!/bin/bash +set -euo pipefail + +# This wrapper is installed only into the GCP service's root-controlled bind. +# Ignore per-invocation mode and endpoint overrides so a service child cannot +# select the VPS/local backend or a different database credential context. +SCRIPT_PATH=${BASH_SOURCE[0]} +SCRIPT_DIR=${SCRIPT_PATH%/*} +if [ "$SCRIPT_DIR" = "$SCRIPT_PATH" ]; then + SCRIPT_DIR=. +fi +SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)" +CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py" + +for argument in "$@"; do + case "$argument" in + --host|--host=*|--port|--port=*|--db|--db=*|--canonical-db|--canonical-db=*|\ + --user|--user=*|--password-secret|--password-secret=*|--project|--project=*|\ + --credential-mode|--credential-mode=*) + echo "teleo-kb: GCP runtime identity and endpoint overrides are forbidden" >&2 + exit 64 + ;; + esac +done + +if [ ! -x "$CLOUDSQL_TOOL" ]; then + echo "teleo-kb: reviewed GCP Cloud SQL tool is unavailable" >&2 + exit 69 +fi +if [ ! -x /usr/bin/python3 ] || [ ! -x /usr/bin/psql ]; then + echo "teleo-kb: required GCP runtime dependency is unavailable" >&2 + exit 69 +fi + +exec /usr/bin/env -i \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + LANG=C LC_ALL=C \ + PYTHONNOUSERSITE=1 PYTHONSAFEPATH=1 \ + CLOUDSDK_CONFIG=/usr/local/libexec/livingip/leoclean-kb/gcloud-config \ + PGSSLMODE=verify-ca \ + PGSSLROOTCERT=/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem \ + TELEO_KB_MODE=cloudsql \ + TELEO_GCP_METADATA_ONLY=1 \ + TELEO_GCP_PROJECT=teleo-501523 \ + TELEO_CLOUDSQL_HOST=10.61.0.3 \ + TELEO_CLOUDSQL_PORT=5432 \ + TELEO_CLOUDSQL_DB=teleo_canonical \ + TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical \ + TELEO_CLOUDSQL_USER=leoclean_kb_runtime \ + TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \ + TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \ + /usr/bin/python3 -I "$CLOUDSQL_TOOL" "$@" diff --git a/ops/gcp-teleo-pgvector-standby-server-ca.pem b/ops/gcp-teleo-pgvector-standby-server-ca.pem new file mode 100644 index 0000000..e31490d --- /dev/null +++ b/ops/gcp-teleo-pgvector-standby-server-ca.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDcTCCAlmgAwIBAgIBADANBgkqhkiG9w0BAQsFADBwMS0wKwYDVQQuEyQzNTcw +NmEzOC03MGRjLTQ3YTQtOWExOS04NGMxYTk0YzQ3ZjMxHDAaBgNVBAMTE0Nsb3Vk +IFNRTCBTZXJ2ZXIgQ0ExFDASBgNVBAoTC0dvb2dsZSwgSW5jMQswCQYDVQQGEwJV +UzAeFw0yNjA3MDcwOTM3NTJaFw0zNjA3MDQwOTM4NTJaMHAxLTArBgNVBC4TJDM1 +NzA2YTM4LTcwZGMtNDdhNC05YTE5LTg0YzFhOTRjNDdmMzEcMBoGA1UEAxMTQ2xv +dWQgU1FMIFNlcnZlciBDQTEUMBIGA1UEChMLR29vZ2xlLCBJbmMxCzAJBgNVBAYT +AlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0HwhrUzFIi9bqPbC +9FQjCalxEVYsu7tw+YzBtUaKOy+u/Y1e6TKJzFng/9wS8pXgY1Ul1/mn1S205yKR +5ydEESq7UL4pp7onSb2vGeu4NDTXDbcimZz++35dPbhFw7wtm1QSRMBHsONEDGaS +7/qIB3EN79SS9GxbSUZNsZOAn4DM9RSahtQrXa6NLlP7I9gPxvqEKRjFqUcBkWsE +Qzoz/yyxYQybZQo1UR9XCbGWFphi85cW15ITpfs7czoOVUk39bPNJKAVQFRqPUWK +xBC+1pHb7fe7xYk5j3jJbtoL6ixunQIFDCGRCy7/SEfIj5dWriDajVHwctilUmu1 +Xy/A0QIDAQABoxYwFDASBgNVHRMBAf8ECDAGAQH/AgEAMA0GCSqGSIb3DQEBCwUA +A4IBAQBi8MoiT4kJAut4mLPibKldRB9s2Of7Jw8PmKNS50BIOZf8zrSrNMX5yC2i +/y9nmc8k9OCIYbzC6F/mTACmtHM1phoBtPYDjWYp+YS9cdo0HHGU2hwEP7PTfEOg +11ua2J1gjFbgVZVb7qNGV4n1QjCcAh1hWnJ2gLa55hiZBGZ0fuHTwBagWwRtom+u +YdD1d1NETInnCxnhzu7q+r0sjFUJABIj4qU/q/uVMHaAuy4Zy4PMctQofKSJ4ixf +vbU2/Sm7cbzgzcg2TJwpSqdDM/EdDMi5JtmMyZnRoKLZjH3CYI1091dyTBdk8RhK +MB4pqjJurXS8xKeA9dQjuvnIQvot +-----END CERTIFICATE----- diff --git a/ops/gcp_leoclean_runtime_role.sql b/ops/gcp_leoclean_runtime_role.sql index da884c8..5962180 100644 --- a/ops/gcp_leoclean_runtime_role.sql +++ b/ops/gcp_leoclean_runtime_role.sql @@ -1,7 +1,8 @@ \set ON_ERROR_STOP on --- One-time GCP staging provisioning for Leo's Cloud SQL identity. The password --- is supplied only through the process environment and is never stored here. +-- One-time GCP staging provisioning for Leo's Cloud SQL identity. Run this +-- only through ops/provision_gcp_leoclean_runtime_role.sh; psql's \password +-- command encrypts the supplied value client-side, so cleartext is never SQL. -- -- The login can read the canonical allowlist and can stage a pending proposal -- through one narrowly-scoped SECURITY DEFINER function. It cannot inherit or @@ -24,19 +25,34 @@ alter role leoclean_kb_stage_owner with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1 password null; --- Rotate the credential only after an idempotent rerun has disabled any --- pre-existing LOGIN role. This ordering has an intentional autocommit --- boundary: interruption can leave NOLOGIN, never a still-enabled old role. -\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD -select format('alter role leoclean_kb_runtime password %L', :'runtime_password') -\gexec +-- Snapshot every non-superuser principal connected to either scoped role before +-- removing the direct membership edges. Existing sessions can retain an +-- already-selected SET ROLE identity after NOLOGIN/revocation, so all other +-- connected sessions must be terminated before password or privilege work. +create temporary table leoclean_fenced_roles ( + role_oid oid primary key, + role_name name unique not null +) on commit preserve rows; -alter role leoclean_kb_runtime reset all; -alter role leoclean_kb_stage_owner reset all; +with recursive connected(role_oid) as ( + values + ('leoclean_kb_runtime'::pg_catalog.regrole::oid), + ('leoclean_kb_stage_owner'::pg_catalog.regrole::oid) + union + select membership.member + from connected + join pg_catalog.pg_auth_members membership + on membership.roleid = connected.role_oid +) +insert into pg_temp.leoclean_fenced_roles (role_oid, role_name) +select role_row.oid, role_row.rolname + from connected + join pg_catalog.pg_roles role_row on role_row.oid = connected.role_oid + where not role_row.rolsuper; --- NOINHERIT does not prevent SET ROLE. Remove every membership edge in either --- direction so neither scoped principal can reach, or be reached through, a --- broader role left behind by an earlier deployment. +-- NOINHERIT does not prevent SET ROLE. Remove every direct membership edge in +-- either direction before terminating the snapshotted sessions, closing the +-- race for newly connected membership principals. select format('revoke %I from %I', granted_role.rolname, member_role.rolname) from pg_catalog.pg_auth_members membership join pg_catalog.pg_roles granted_role on granted_role.oid = membership.roleid @@ -45,11 +61,62 @@ select format('revoke %I from %I', granted_role.rolname, member_role.rolname) or member_role.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') \gexec -revoke all privileges on database teleo_canonical from leoclean_kb_runtime; -revoke all privileges on database teleo_canonical from leoclean_kb_stage_owner; -revoke all privileges on database teleo_kb from leoclean_kb_runtime; -revoke all privileges on database teleo_kb from leoclean_kb_stage_owner; -grant connect on database teleo_canonical to leoclean_kb_runtime; +do $session_fence$ +declare + target record; + remaining text; +begin + for target in + select activity.pid + from pg_catalog.pg_stat_activity activity + join pg_temp.leoclean_fenced_roles fenced on fenced.role_name = activity.usename + where activity.pid <> pg_catalog.pg_backend_pid() + and activity.backend_type = 'client backend' + loop + if not pg_catalog.pg_terminate_backend(target.pid, 5000) then + raise exception 'could not terminate a pre-existing scoped-role session'; + end if; + end loop; + + select pg_catalog.string_agg(activity.pid::text, ',' order by activity.pid) + into remaining + from pg_catalog.pg_stat_activity activity + join pg_temp.leoclean_fenced_roles fenced on fenced.role_name = activity.usename + where activity.pid <> pg_catalog.pg_backend_pid() + and activity.backend_type = 'client backend'; + if remaining is not null then + raise exception 'pre-existing scoped-role sessions remain after termination'; + end if; + + select pg_catalog.string_agg(prepared.gid, ',' order by prepared.gid) + into remaining + from pg_catalog.pg_prepared_xacts prepared + join pg_temp.leoclean_fenced_roles fenced on fenced.role_name = prepared.owner; + if remaining is not null then + raise exception 'scoped-role prepared transactions must be resolved before provisioning'; + end if; +end +$session_fence$; + +-- This is deliberately after the autocommitted NOLOGIN/session/membership +-- fence. Interruption can leave a disabled role, never a still-enabled old +-- login or an old connected session. The wrapper supplies the two prompts. +\password leoclean_kb_runtime + +alter role leoclean_kb_runtime reset all; +alter role leoclean_kb_stage_owner reset all; +drop table pg_temp.leoclean_fenced_roles; + +select pg_catalog.format( + 'alter role %I in database %I reset all', + role_row.rolname, + database_row.datname + ) + from pg_catalog.pg_db_role_setting setting_row + join pg_catalog.pg_roles role_row on role_row.oid = setting_row.setrole + join pg_catalog.pg_database database_row on database_row.oid = setting_row.setdatabase + where role_row.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') +\gexec -- PostgreSQL normally grants TEMP to PUBLIC. A role-specific REVOKE cannot -- override that inherited grant, while REVOKE TEMP FROM PUBLIC would be a @@ -77,6 +144,47 @@ alter role leoclean_kb_runtime in database teleo_canonical begin; +-- CONNECT granted to PUBLIC cannot be denied to one role with a role-specific +-- REVOKE. Preserve every existing non-scoped principal's effective CONNECT +-- set as explicit ACLs, remove the cluster-wide PUBLIC grant from every +-- database (including templates), and then grant the runtime its sole target. +-- This runs in the same transaction as every remaining privilege/topology +-- assertion, so a later failure restores the original ACLs atomically. +create temporary table leoclean_preserved_database_connect ( + role_name name not null, + database_name name not null, + primary key (role_name, database_name) +) on commit drop; + +insert into pg_temp.leoclean_preserved_database_connect (role_name, database_name) +select role_row.rolname, database_row.datname + from pg_catalog.pg_roles role_row + cross join pg_catalog.pg_database database_row + where role_row.rolname not in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and pg_catalog.has_database_privilege(role_row.oid, database_row.oid, 'CONNECT'); + +select pg_catalog.format('revoke connect on database %I from public', database_row.datname) + from pg_catalog.pg_database database_row +\gexec + +select pg_catalog.format( + 'revoke all privileges on database %I from leoclean_kb_runtime, leoclean_kb_stage_owner', + database_row.datname + ) + from pg_catalog.pg_database database_row +\gexec + +select pg_catalog.format( + 'grant connect on database %I to %I', + preserved.database_name, + preserved.role_name + ) + from pg_temp.leoclean_preserved_database_connect preserved + order by preserved.database_name, preserved.role_name +\gexec + +grant connect on database teleo_canonical to leoclean_kb_runtime; + -- The deploy principal needs temporary ownership authority to replace an -- existing function on idempotent reruns. This membership is transaction-local -- and is revoked before verification/commit. @@ -95,6 +203,114 @@ revoke all privileges on all functions in schema public, kb_stage revoke all privileges on all procedures in schema public, kb_stage from leoclean_kb_runtime, leoclean_kb_stage_owner; +-- Built-in defaults also grant PUBLIC EXECUTE on newly created application +-- routines. Preserve every existing non-scoped role's effective access, +-- remove PUBLIC/scoped execution from the complete application routine set, +-- and restore the inventory explicitly. The reviewed staging function is +-- normalized again after CREATE OR REPLACE below. +create temporary table leoclean_preserved_app_routine_execute ( + role_name name not null, + function_oid oid not null, + primary key (role_name, function_oid) +) on commit drop; + +insert into pg_temp.leoclean_preserved_app_routine_execute (role_name, function_oid) +select role_row.rolname, function_row.oid + from pg_catalog.pg_roles role_row + cross join pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where role_row.rolname not in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and namespace.nspname in ('public', 'kb_stage') + and pg_catalog.has_function_privilege(role_row.oid, function_row.oid, 'EXECUTE'); + +select pg_catalog.format( + 'revoke execute on %s %I.%I(%s) from public, leoclean_kb_runtime, leoclean_kb_stage_owner', + case when function_row.prokind = 'p' then 'procedure' else 'function' end, + namespace.nspname, + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid) + ) + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname in ('public', 'kb_stage') +\gexec + +select pg_catalog.format( + 'grant execute on %s %I.%I(%s) to %I', + case when function_row.prokind = 'p' then 'procedure' else 'function' end, + namespace.nspname, + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid), + preserved.role_name + ) + from pg_temp.leoclean_preserved_app_routine_execute preserved + join pg_catalog.pg_proc function_row on function_row.oid = preserved.function_oid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + order by namespace.nspname, function_row.proname, preserved.role_name +\gexec + +select pg_catalog.format( + 'revoke all privileges on large object %s from leoclean_kb_runtime, leoclean_kb_stage_owner', + metadata.oid + ) + from pg_catalog.pg_largeobject_metadata metadata +\gexec + +-- Default PostgreSQL grants EXECUTE on built-in large-object mutators to +-- PUBLIC, which would let the runtime persist data despite all relation ACLs. +-- Snapshot and explicitly preserve every existing non-scoped role's effective +-- access, then remove PUBLIC and both scoped roles from every mutator overload. +create temporary table leoclean_preserved_lo_execute ( + role_name name not null, + function_oid oid not null, + primary key (role_name, function_oid) +) on commit drop; + +insert into pg_temp.leoclean_preserved_lo_execute (role_name, function_oid) +select role_row.rolname, function_row.oid + from pg_catalog.pg_roles role_row + cross join pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where role_row.rolname not in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + and namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and pg_catalog.has_function_privilege(role_row.oid, function_row.oid, 'EXECUTE'); + +select pg_catalog.format( + 'revoke execute on function pg_catalog.%I(%s) from public, leoclean_kb_runtime, leoclean_kb_stage_owner', + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid) + ) + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) +\gexec + +select pg_catalog.format( + 'grant execute on function pg_catalog.%I(%s) to %I', + function_row.proname, + pg_catalog.pg_get_function_identity_arguments(function_row.oid), + preserved.role_name + ) + from pg_temp.leoclean_preserved_lo_execute preserved + join pg_catalog.pg_proc function_row on function_row.oid = preserved.function_oid + order by function_row.proname, preserved.role_name +\gexec + +select pg_catalog.format( + 'revoke all privileges on parameter %I from leoclean_kb_runtime, leoclean_kb_stage_owner', + parameter_acl.parname + ) + from pg_catalog.pg_parameter_acl parameter_acl +\gexec + -- Table-level REVOKE does not clear column ACLs. Clear every column ACL for both -- principals before rebuilding the exact allowlist below. select format( @@ -310,6 +526,264 @@ begin raise exception 'scoped Leo roles retain role memberships: %', unexpected; end if; + if not exists ( + select 1 + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' + and relation.relkind = 'r' + and relation.relpersistence = 'p' + and not relation.relrowsecurity + and not relation.relforcerowsecurity + ) then + raise exception 'kb_stage.kb_proposals is not the reviewed permanent base table'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, relation.relkind), + ', ' + order by namespace.nspname, relation.relname + ) + into unexpected + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where (namespace.nspname, relation.relname) in ( + ('public', 'agents'), ('public', 'claims'), ('public', 'claim_evidence'), + ('public', 'claim_edges'), ('public', 'sources'), ('public', 'personas'), + ('public', 'strategies'), ('public', 'beliefs'), ('public', 'blindspots'), + ('public', 'agent_roles'), ('public', 'peer_models'), ('public', 'behavioral_rules'), + ('public', 'contributor_rules'), ('public', 'reasoning_tools'), + ('public', 'governance_gates'), ('kb_stage', 'kb_proposals') + ) + and relation.relkind <> 'r'; + if unexpected is not null then + raise exception 'Leo read allowlist contains a non-base relation: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s->%s', inheritance.inhparent::pg_catalog.regclass, inheritance.inhrelid::pg_catalog.regclass), + ', ' + ) + into unexpected + from pg_catalog.pg_inherits inheritance + where inheritance.inhparent in ( + 'public.agents'::pg_catalog.regclass, 'public.claims'::pg_catalog.regclass, + 'public.claim_evidence'::pg_catalog.regclass, 'public.claim_edges'::pg_catalog.regclass, + 'public.sources'::pg_catalog.regclass, 'public.personas'::pg_catalog.regclass, + 'public.strategies'::pg_catalog.regclass, 'public.beliefs'::pg_catalog.regclass, + 'public.blindspots'::pg_catalog.regclass, 'public.agent_roles'::pg_catalog.regclass, + 'public.peer_models'::pg_catalog.regclass, 'public.behavioral_rules'::pg_catalog.regclass, + 'public.contributor_rules'::pg_catalog.regclass, 'public.reasoning_tools'::pg_catalog.regclass, + 'public.governance_gates'::pg_catalog.regclass, 'kb_stage.kb_proposals'::pg_catalog.regclass + ) + or inheritance.inhrelid in ( + 'public.agents'::pg_catalog.regclass, 'public.claims'::pg_catalog.regclass, + 'public.claim_evidence'::pg_catalog.regclass, 'public.claim_edges'::pg_catalog.regclass, + 'public.sources'::pg_catalog.regclass, 'public.personas'::pg_catalog.regclass, + 'public.strategies'::pg_catalog.regclass, 'public.beliefs'::pg_catalog.regclass, + 'public.blindspots'::pg_catalog.regclass, 'public.agent_roles'::pg_catalog.regclass, + 'public.peer_models'::pg_catalog.regclass, 'public.behavioral_rules'::pg_catalog.regclass, + 'public.contributor_rules'::pg_catalog.regclass, 'public.reasoning_tools'::pg_catalog.regclass, + 'public.governance_gates'::pg_catalog.regclass, 'kb_stage.kb_proposals'::pg_catalog.regclass + ); + if unexpected is not null then + raise exception 'Leo read allowlist has inheritance edges: %', unexpected; + end if; + + with expected(ordinal, attname, typname, not_null, has_default, collated) as ( + values + (1, 'id', 'uuid', true, true, false), + (2, 'proposal_type', 'text', true, false, true), + (3, 'status', 'text', true, true, true), + (4, 'proposed_by_handle', 'text', false, false, true), + (5, 'proposed_by_agent_id', 'uuid', false, false, false), + (6, 'channel', 'text', true, true, true), + (7, 'source_ref', 'text', false, false, true), + (8, 'rationale', 'text', true, false, true), + (9, 'payload', 'jsonb', true, false, false), + (10, 'reviewed_by_handle', 'text', false, false, true), + (11, 'reviewed_by_agent_id', 'uuid', false, false, false), + (12, 'reviewed_at', 'timestamptz', false, false, false), + (13, 'review_note', 'text', false, false, true), + (14, 'applied_by_handle', 'text', false, false, true), + (15, 'applied_by_agent_id', 'uuid', false, false, false), + (16, 'applied_at', 'timestamptz', false, false, false), + (17, 'created_at', 'timestamptz', true, true, false), + (18, 'updated_at', 'timestamptz', true, true, false) + ), actual as ( + select attribute.attnum::int as ordinal, + attribute.attname, + type_row.typname, + attribute.attnotnull as not_null, + attribute.atthasdef as has_default, + attribute.attidentity, + attribute.attgenerated, + attribute.atttypmod, + type_namespace.nspname as type_namespace, + attribute.attcollation = 'pg_catalog.default'::pg_catalog.regcollation as collated + from pg_catalog.pg_attribute attribute + join pg_catalog.pg_type type_row on type_row.oid = attribute.atttypid + join pg_catalog.pg_namespace type_namespace on type_namespace.oid = type_row.typnamespace + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attnum > 0 + and not attribute.attisdropped + ) + select pg_catalog.string_agg(coalesce(expected.attname, actual.attname), ', ' order by coalesce(expected.ordinal, actual.ordinal)) + into unexpected + from expected + full join actual using (ordinal, attname) + where expected.ordinal is null + or actual.ordinal is null + or expected.typname is distinct from actual.typname + or expected.not_null is distinct from actual.not_null + or expected.has_default is distinct from actual.has_default + or expected.collated is distinct from actual.collated + or actual.attidentity <> '' + or actual.attgenerated <> '' + or actual.atttypmod <> -1 + or actual.type_namespace <> 'pg_catalog'; + if unexpected is not null then + raise exception 'kb_stage.kb_proposals column contract drifted: %', unexpected; + end if; + + if exists ( + select 1 + from pg_catalog.pg_trigger trigger_row + where trigger_row.tgrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and not trigger_row.tgisinternal + ) or exists ( + select 1 + from pg_catalog.pg_rewrite rewrite_row + where rewrite_row.ev_class = 'kb_stage.kb_proposals'::pg_catalog.regclass + ) or exists ( + select 1 + from pg_catalog.pg_policy policy_row + where policy_row.polrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + ) or exists ( + select 1 + from pg_catalog.pg_attribute attribute + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attnum > 0 + and not attribute.attisdropped + and attribute.attgenerated <> '' + ) then + raise exception 'kb_stage.kb_proposals has unsafe trigger/rule/RLS/generated topology'; + end if; + + with expected(attname, expression) as ( + values + ('id', 'gen_random_uuid()'), + ('status', '''pending_review''::text'), + ('channel', '''cli''::text'), + ('created_at', 'now()'), + ('updated_at', 'now()') + ), actual as ( + select attribute.attname, + pg_catalog.pg_get_expr(default_row.adbin, default_row.adrelid) as expression + from pg_catalog.pg_attribute attribute + join pg_catalog.pg_attrdef default_row + on default_row.adrelid = attribute.attrelid + and default_row.adnum = attribute.attnum + where attribute.attrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and attribute.attnum > 0 + and not attribute.attisdropped + ) + select pg_catalog.string_agg(coalesce(expected.attname, actual.attname), ', ') + into unexpected + from expected + full join actual using (attname) + where expected.expression is distinct from actual.expression; + if unexpected is not null then + raise exception 'kb_stage.kb_proposals default topology drifted: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s:%s', dependency.refclassid::pg_catalog.regclass, dependency.refobjid), + ', ' + ) + into unexpected + from pg_catalog.pg_attrdef default_row + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + where default_row.adrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and ( + dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + and dependency.refobjid not in ( + 'pg_catalog.gen_random_uuid()'::pg_catalog.regprocedure, + 'pg_catalog.now()'::pg_catalog.regprocedure + ) + or dependency.refclassid = 'pg_catalog.pg_type'::pg_catalog.regclass + and not exists ( + select 1 + from pg_catalog.pg_type type_row + join pg_catalog.pg_namespace namespace on namespace.oid = type_row.typnamespace + where type_row.oid = dependency.refobjid + and namespace.nspname = 'pg_catalog' + ) + or dependency.refclassid = 'pg_catalog.pg_operator'::pg_catalog.regclass + and not exists ( + select 1 + from pg_catalog.pg_operator operator_row + join pg_catalog.pg_namespace namespace on namespace.oid = operator_row.oprnamespace + where operator_row.oid = dependency.refobjid + and namespace.nspname = 'pg_catalog' + ) + or dependency.refclassid = 'pg_catalog.pg_collation'::pg_catalog.regclass + and not exists ( + select 1 + from pg_catalog.pg_collation collation_row + join pg_catalog.pg_namespace namespace on namespace.oid = collation_row.collnamespace + where collation_row.oid = dependency.refobjid + and namespace.nspname = 'pg_catalog' + ) + ); + if unexpected is not null then + raise exception 'kb_stage.kb_proposals defaults depend on unreviewed objects: %', unexpected; + end if; + + if ( + select pg_catalog.count(*) <> 6 + or pg_catalog.count(*) filter ( + where (constraint_row.conname, constraint_row.contype) in ( + ('kb_proposals_pkey', 'p'), + ('kb_proposals_proposal_type_check', 'c'), + ('kb_proposals_status_check', 'c'), + ('kb_proposals_applied_by_agent_id_fkey', 'f'), + ('kb_proposals_proposed_by_agent_id_fkey', 'f'), + ('kb_proposals_reviewed_by_agent_id_fkey', 'f') + ) + ) <> 6 + from pg_catalog.pg_constraint constraint_row + where constraint_row.conrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + ) or exists ( + select 1 + from pg_catalog.pg_constraint constraint_row + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_constraint'::pg_catalog.regclass + and dependency.objid = constraint_row.oid + and dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + join pg_catalog.pg_proc function_row on function_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where constraint_row.conrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and namespace.nspname <> 'pg_catalog' + ) or exists ( + select 1 + from pg_catalog.pg_index index_row + where index_row.indrelid = 'kb_stage.kb_proposals'::pg_catalog.regclass + and ( + index_row.indexprs is not null + or ( + index_row.indpred is not null + and pg_catalog.pg_get_expr(index_row.indpred, index_row.indrelid) + <> '(proposed_by_handle IS NOT NULL)' + ) + ) + ) then + raise exception 'kb_stage.kb_proposals constraint/index topology drifted'; + end if; + select pg_catalog.string_agg( pg_catalog.format( '%I:%s:%s:grantable=%s', @@ -330,8 +804,7 @@ begin ) ) as acl join pg_catalog.pg_roles grantee on grantee.oid = acl.grantee - where database_row.datname in ('teleo_canonical', 'teleo_kb') - and grantee.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') + where grantee.rolname in ('leoclean_kb_runtime', 'leoclean_kb_stage_owner') and not ( database_row.datname = 'teleo_canonical' and grantee.rolname = 'leoclean_kb_runtime' @@ -342,11 +815,86 @@ begin raise exception 'scoped Leo roles retain unexpected direct database ACLs: %', unexpected; end if; - if has_database_privilege('leoclean_kb_runtime', 'teleo_canonical', 'CREATE') - or has_database_privilege('leoclean_kb_runtime', 'teleo_kb', 'CREATE') - or has_database_privilege('leoclean_kb_stage_owner', 'teleo_canonical', 'CREATE') - or has_database_privilege('leoclean_kb_stage_owner', 'teleo_kb', 'CREATE') then - raise exception 'a scoped Leo role unexpectedly has database CREATE'; + select pg_catalog.string_agg(database_row.datname, ', ' order by database_row.datname) + into unexpected + from pg_catalog.pg_database database_row + where database_row.datname <> 'teleo_canonical' + and has_database_privilege('leoclean_kb_runtime', database_row.oid, 'CONNECT'); + if unexpected is not null then + raise exception 'runtime CONNECT reaches a noncanonical database: %', unexpected; + end if; + + select pg_catalog.string_agg(database_row.datname, ', ' order by database_row.datname) + into unexpected + from pg_catalog.pg_database database_row + where has_database_privilege('leoclean_kb_stage_owner', database_row.oid, 'CONNECT'); + if unexpected is not null then + raise exception 'stage owner CONNECT reaches a database: %', unexpected; + end if; + + select pg_catalog.string_agg(database_row.datname, ', ' order by database_row.datname) + into unexpected + from pg_catalog.pg_database database_row + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where acl.grantee = 0 + and acl.privilege_type = 'CONNECT'; + if unexpected is not null then + raise exception 'PUBLIC retains database CONNECT: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%s:%s', setting_row.setdatabase, setting_row.setconfig), + ', ' + ) + into unexpected + from pg_catalog.pg_db_role_setting setting_row + where setting_row.setrole in (runtime_oid, owner_oid) + and not ( + setting_row.setrole = runtime_oid + and setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = 'teleo_canonical' + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ); + if unexpected is not null or not exists ( + select 1 + from pg_catalog.pg_db_role_setting setting_row + where setting_row.setrole = runtime_oid + and setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = 'teleo_canonical' + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ) then + raise exception 'scoped Leo role settings differ from the reviewed contract'; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I:%s', database_row.datname, scoped_role.name), + ', ' + order by database_row.datname, scoped_role.name + ) + into unexpected + from pg_catalog.pg_database database_row + cross join (values ('leoclean_kb_runtime'), ('leoclean_kb_stage_owner')) scoped_role(name) + where has_database_privilege(scoped_role.name, database_row.oid, 'CREATE'); + if unexpected is not null then + raise exception 'a scoped Leo role unexpectedly has database CREATE: %', unexpected; end if; select pg_catalog.string_agg( @@ -382,6 +930,75 @@ begin raise exception 'leoclean_kb_stage_owner owns unexpected database objects: %', unexpected; end if; + select pg_catalog.string_agg(metadata.oid::text, ', ' order by metadata.oid) + into unexpected + from pg_catalog.pg_largeobject_metadata metadata + where metadata.lomowner in (runtime_oid, owner_oid) + or exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(metadata.lomacl, pg_catalog.acldefault('L', metadata.lomowner)) + ) acl + where acl.grantee in (0, runtime_oid, owner_oid) + and acl.privilege_type in ('SELECT', 'UPDATE') + ); + if unexpected is not null then + raise exception 'scoped Leo roles retain large-object ownership or ACL reachability: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I(%s)', function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid)), + ', ' + ) + into unexpected + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and ( + has_function_privilege('leoclean_kb_runtime', function_row.oid, 'EXECUTE') + or has_function_privilege('leoclean_kb_stage_owner', function_row.oid, 'EXECUTE') + ); + if unexpected is not null then + raise exception 'PUBLIC or inherited large-object mutators remain executable: %', unexpected; + end if; + + select pg_catalog.string_agg( + pg_catalog.format('%I(%s)', function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid)), + ', ' + order by function_row.proname, pg_catalog.pg_get_function_identity_arguments(function_row.oid) + ) + into unexpected + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + cross join lateral pg_catalog.aclexplode( + coalesce(function_row.proacl, pg_catalog.acldefault('f', function_row.proowner)) + ) acl + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and acl.grantee = 0 + and acl.privilege_type = 'EXECUTE'; + if unexpected is not null then + raise exception 'PUBLIC retains large-object mutator EXECUTE: %', unexpected; + end if; + + select pg_catalog.string_agg(parameter_acl.parname, ', ' order by parameter_acl.parname) + into unexpected + from pg_catalog.pg_parameter_acl parameter_acl + where has_parameter_privilege('leoclean_kb_runtime', parameter_acl.parname, 'SET') + or has_parameter_privilege('leoclean_kb_runtime', parameter_acl.parname, 'ALTER SYSTEM') + or has_parameter_privilege('leoclean_kb_stage_owner', parameter_acl.parname, 'SET') + or has_parameter_privilege('leoclean_kb_stage_owner', parameter_acl.parname, 'ALTER SYSTEM'); + if unexpected is not null then + raise exception 'scoped Leo roles retain parameter privileges: %', unexpected; + end if; + if has_schema_privilege('leoclean_kb_runtime', 'public', 'CREATE') or has_schema_privilege('leoclean_kb_runtime', 'kb_stage', 'CREATE') or has_schema_privilege('leoclean_kb_stage_owner', 'public', 'CREATE') @@ -698,137 +1315,9 @@ begin end $verification$; -commit; - --- The legacy restore database is not an audit fallback for the scoped runtime. --- Strip direct ACLs from both scoped roles and fail if PUBLIC/group grants still --- make any teleo_restore relation reachable. -\connect teleo_kb - -begin; - -select format('revoke all on schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) - from pg_catalog.pg_namespace namespace - where namespace.nspname = 'teleo_restore' -\gexec - -select format('revoke all privileges on all tables in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) - from pg_catalog.pg_namespace namespace - where namespace.nspname = 'teleo_restore' -\gexec - -select format('revoke all privileges on all sequences in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) - from pg_catalog.pg_namespace namespace - where namespace.nspname = 'teleo_restore' -\gexec - -select format('revoke all privileges on all functions in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) - from pg_catalog.pg_namespace namespace - where namespace.nspname = 'teleo_restore' -\gexec - -select format('revoke all privileges on all procedures in schema %I from leoclean_kb_runtime, leoclean_kb_stage_owner', namespace.nspname) - from pg_catalog.pg_namespace namespace - where namespace.nspname = 'teleo_restore' -\gexec - -select format( - 'revoke all privileges (%s) on table %I.%I from %I', - pg_catalog.string_agg(pg_catalog.quote_ident(attribute.attname), ', ' order by attribute.attnum), - namespace.nspname, - relation.relname, - target.rolname - ) - from pg_catalog.pg_class relation - join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace - join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid - cross join ( - values ('leoclean_kb_runtime'), ('leoclean_kb_stage_owner') - ) as target(rolname) - where namespace.nspname = 'teleo_restore' - and relation.relkind in ('r', 'p', 'v', 'm', 'f') - and attribute.attnum > 0 - and not attribute.attisdropped - group by namespace.nspname, relation.relname, target.rolname -\gexec - -do $audit_verification$ -declare - unexpected text; - scoped_role text; -begin - foreach scoped_role in array array['leoclean_kb_runtime', 'leoclean_kb_stage_owner'] loop - if to_regnamespace('teleo_restore') is not null - and (has_schema_privilege(scoped_role, 'teleo_restore', 'USAGE') - or has_schema_privilege(scoped_role, 'teleo_restore', 'CREATE')) then - raise exception '% unexpectedly has teleo_restore schema access', scoped_role; - end if; - - select pg_catalog.string_agg( - pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name), - ', ' - order by relation.relname, privilege.name - ) - into unexpected - from pg_catalog.pg_class relation - join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace - cross join ( - values ('SELECT'), ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER') - ) privilege(name) - where namespace.nspname = 'teleo_restore' - and relation.relkind in ('r', 'p', 'v', 'm', 'f') - and has_table_privilege(scoped_role, relation.oid, privilege.name); - if unexpected is not null then - raise exception '% unexpectedly has teleo_restore table access: %', scoped_role, unexpected; - end if; - - select pg_catalog.string_agg( - pg_catalog.format('%I.%I.%I:%s', namespace.nspname, relation.relname, attribute.attname, privilege.name), - ', ' - order by relation.relname, attribute.attnum, privilege.name - ) - into unexpected - from pg_catalog.pg_class relation - join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace - join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid - cross join (values ('SELECT'), ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) - where namespace.nspname = 'teleo_restore' - and relation.relkind in ('r', 'p', 'v', 'm', 'f') - and attribute.attnum > 0 - and not attribute.attisdropped - and has_column_privilege(scoped_role, relation.oid, attribute.attnum, privilege.name); - if unexpected is not null then - raise exception '% unexpectedly has teleo_restore column access: %', scoped_role, unexpected; - end if; - - select pg_catalog.string_agg( - pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name), - ', ' - order by relation.relname, privilege.name - ) - into unexpected - from pg_catalog.pg_class relation - join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace - cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) - where namespace.nspname = 'teleo_restore' - and relation.relkind = 'S' - and has_sequence_privilege(scoped_role, relation.oid, privilege.name); - if unexpected is not null then - raise exception '% unexpectedly has teleo_restore sequence access: %', scoped_role, unexpected; - end if; - end loop; -end -$audit_verification$; - -commit; - -\connect teleo_canonical - --- LOGIN is the last state change. If this verification fails, the transaction --- rolls back and the role remains NOLOGIN while prior least-privilege grants --- stay inert for direct authentication. -begin; - +-- LOGIN is the last state change in the same transaction as every database ACL, +-- canonical privilege, large-object, and topology assertion. Any failure rolls +-- back the complete privilege migration and leaves the role NOLOGIN. alter role leoclean_kb_runtime with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8; @@ -838,14 +1327,16 @@ begin select 1 from pg_catalog.pg_roles role_row where role_row.rolname = 'leoclean_kb_runtime' - and (not role_row.rolcanlogin - or role_row.rolsuper - or role_row.rolcreatedb - or role_row.rolcreaterole - or role_row.rolinherit - or role_row.rolreplication - or role_row.rolbypassrls - or role_row.rolconnlimit <> 8) + and ( + not role_row.rolcanlogin + or role_row.rolsuper + or role_row.rolcreatedb + or role_row.rolcreaterole + or role_row.rolinherit + or role_row.rolreplication + or role_row.rolbypassrls + or role_row.rolconnlimit <> 8 + ) ) then raise exception 'leoclean_kb_runtime final login attributes are unsafe'; end if; @@ -853,32 +1344,3 @@ end $login_verification$; commit; - -select jsonb_build_object( - 'role', rolname, - 'superuser', rolsuper, - 'create_db', rolcreatedb, - 'create_role', rolcreaterole, - 'inherit', rolinherit, - 'replication', rolreplication, - 'bypass_rls', rolbypassrls, - 'has_role_memberships', exists ( - select 1 - from pg_catalog.pg_auth_members membership - where membership.roleid = pg_catalog.pg_roles.oid - or membership.member = pg_catalog.pg_roles.oid - ), - 'can_select_claims', has_table_privilege(rolname, 'public.claims', 'SELECT'), - 'can_select_proposals', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'SELECT'), - 'can_insert_proposals_directly', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'INSERT'), - 'can_connect_legacy_database', has_database_privilege(rolname, 'teleo_kb', 'CONNECT'), - 'can_create_database_objects', has_database_privilege(rolname, 'teleo_canonical', 'CREATE'), - 'can_create_temp_objects', has_database_privilege(rolname, 'teleo_canonical', 'TEMP'), - 'can_stage_proposal', has_function_privilege( - rolname, - 'kb_stage.stage_leoclean_proposal(text,text,text,text,jsonb)', - 'EXECUTE' - ) -)::text -from pg_catalog.pg_roles -where rolname = 'leoclean_kb_runtime'; diff --git a/ops/provision_gcp_leoclean_runtime_role.sh b/ops/provision_gcp_leoclean_runtime_role.sh new file mode 100755 index 0000000..9ff89f8 --- /dev/null +++ b/ops/provision_gcp_leoclean_runtime_role.sh @@ -0,0 +1,150 @@ +#!/bin/bash +# Provision the scoped GCP staging role without placing its cleartext password +# in SQL text, argv, command history, or PostgreSQL statement logs. +set +x +set -euo pipefail +umask 077 +ulimit -c 0 + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PATH +runtime_password=${TELEO_LEOCLEAN_DB_PASSWORD-} +administrator_password=${PGPASSWORD-} +unset TELEO_LEOCLEAN_DB_PASSWORD PGPASSWORD + +script_path=${BASH_SOURCE[0]} +case "$script_path" in + */*) script_directory=${script_path%/*} ;; + *) script_directory=. ;; +esac +SCRIPT_DIR="$(cd -- "$script_directory" && pwd -P)" +SQL_FILE="$SCRIPT_DIR/gcp_leoclean_runtime_role.sql" +SERVER_CA_FILE="$SCRIPT_DIR/gcp-teleo-pgvector-standby-server-ca.pem" +PSQL_BIN=/usr/bin/psql +SETSID_BIN=/usr/bin/setsid +ENV_BIN=/usr/bin/env +SHA256SUM_BIN=/usr/bin/sha256sum +SERVER_CA_SHA256=80701e768f0e1f6b9d621aa0b53f6e851daaa276c6d9a8e51a300fbc015539cb + +assert_root_owned_nonwritable() { + local target="$1" state uid mode + state="$(stat -Lc '%u:%a' -- "$target")" + uid=${state%%:*} + mode=${state#*:} + if [ "$uid" != 0 ] || (( (8#$mode & 8#022) != 0 )); then + echo "Refusing an untrusted PostgreSQL executable path." >&2 + return 1 + fi +} + +assert_trusted_executable() { + local requested="$1" resolved current + [ -x "$requested" ] || { + echo "Required PostgreSQL client is unavailable." >&2 + return 1 + } + resolved="$(readlink -f -- "$requested")" + [ -n "$resolved" ] && [ -f "$resolved" ] && [ -x "$resolved" ] || { + echo "Required PostgreSQL client did not resolve to an executable file." >&2 + return 1 + } + assert_root_owned_nonwritable "$requested" + current="$(dirname "$requested")" + while :; do + assert_root_owned_nonwritable "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done + current="$resolved" + while :; do + assert_root_owned_nonwritable "$current" + [ "$current" = / ] && break + current="$(dirname "$current")" + done +} + +if [ ! -f "$SQL_FILE" ] || [ -L "$SQL_FILE" ] || [ ! -f "$SERVER_CA_FILE" ] || [ -L "$SERVER_CA_FILE" ]; then + echo "Reviewed GCP runtime-role SQL is unavailable." >&2 + exit 66 +fi +assert_trusted_executable "$PSQL_BIN" +assert_trusted_executable "$SETSID_BIN" +assert_trusted_executable "$ENV_BIN" +assert_trusted_executable "$SHA256SUM_BIN" +if [ "$("$SHA256SUM_BIN" "$SERVER_CA_FILE")" != "$SERVER_CA_SHA256 $SERVER_CA_FILE" ]; then + echo "Reviewed Cloud SQL server CA does not match the pinned certificate." >&2 + exit 66 +fi + +if [ -z "$runtime_password" ] || [ "${#runtime_password}" -gt 4096 ]; then + echo "TELEO_LEOCLEAN_DB_PASSWORD must contain a bounded nonempty value." >&2 + exit 64 +fi +case "$runtime_password" in + *$'\n'*|*$'\r'*) + echo "TELEO_LEOCLEAN_DB_PASSWORD must not contain line breaks." >&2 + exit 64 + ;; +esac +if [ -z "$administrator_password" ] || [ "${#administrator_password}" -gt 4096 ]; then + echo "PGPASSWORD must contain a bounded nonempty administrator credential." >&2 + exit 64 +fi +case "$administrator_password" in + *$'\n'*|*$'\r'*) + echo "PGPASSWORD must not contain line breaks." >&2 + exit 64 + ;; +esac +cleanup_password() { + runtime_password= + administrator_password= + unset runtime_password administrator_password +} +provision_pid= +terminate_provisioning() { + local exit_status="$1" + trap - INT TERM + if [[ "$provision_pid" =~ ^[1-9][0-9]*$ ]]; then + kill -TERM -- "-$provision_pid" 2>/dev/null || kill -TERM -- "$provision_pid" 2>/dev/null || true + wait "$provision_pid" 2>/dev/null || true + provision_pid= + fi + cleanup_password + exit "$exit_status" +} +trap cleanup_password EXIT +trap 'terminate_provisioning 130' INT +trap 'terminate_provisioning 143' TERM + +# \password encrypts client-side. --no-password ensures an administrator-login +# prompt cannot consume either of the two runtime-password records from stdin. +# setsid detaches psql from any controlling terminal without --fork, keeping +# its PID as the session/process-group leader so an interruption can terminate +# and reap the complete credential-bearing child before this wrapper returns. +{ + printf '%s\n' "$runtime_password" + printf '%s\n' "$runtime_password" +} | "$ENV_BIN" -i \ + PATH="$PATH" \ + LANG=C LC_ALL=C \ + PYTHONNOUSERSITE=1 PYTHONSAFEPATH=1 \ + PGHOST=10.61.0.3 \ + PGPORT=5432 \ + PGDATABASE=teleo_canonical \ + PGUSER=postgres \ + PGPASSWORD="$administrator_password" \ + PGSSLMODE=verify-ca \ + PGSSLROOTCERT="$SERVER_CA_FILE" \ + "$SETSID_BIN" -- "$PSQL_BIN" \ + --no-psqlrc \ + --no-password \ + --set=ON_ERROR_STOP=1 \ + --file="$SQL_FILE" & +provision_pid=$! +set +e +wait "$provision_pid" +status=$? +set -e +provision_pid= +exit "$status" diff --git a/ops/verify_gcp_leoclean_runtime_permissions.py b/ops/verify_gcp_leoclean_runtime_permissions.py index bdd44dd..94ccc4f 100644 --- a/ops/verify_gcp_leoclean_runtime_permissions.py +++ b/ops/verify_gcp_leoclean_runtime_permissions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 -I """Retain a sanitized least-privilege receipt for the GCP Leo runtime role. This verifier is intentionally service-independent. It must run on the private @@ -19,8 +19,8 @@ import re import stat import subprocess import sys -import tempfile from collections.abc import Callable, Mapping +from contextlib import nullcontext from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -29,8 +29,10 @@ from typing import Any PROJECT_ID = "teleo-501523" PRIVATE_CLOUDSQL_HOST = "10.61.0.3" PRIVATE_CLOUDSQL_PORT = 5432 +EXPECTED_SYSTEM_IDENTIFIER = "7659718422914359312" CANONICAL_DATABASE = "teleo_canonical" LEGACY_DATABASE = "teleo_kb" +TEMPLATE_DATABASE = "template1" RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime" STAGE_OWNER_DATABASE_ROLE = "leoclean_kb_stage_owner" RUNTIME_UNIX_USER = "teleo" @@ -38,6 +40,9 @@ SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-passwor ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password" GCLOUD_BIN = "/usr/bin/gcloud" PSQL_BIN = "/usr/bin/psql" +SERVER_CA_PATH = Path("/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem") +RUNTIME_CLOUDSDK_CONFIG = Path("/usr/local/libexec/livingip/leoclean-kb/gcloud-config") +SERVER_CA_SHA256 = "80701e768f0e1f6b9d621aa0b53f6e851daaa276c6d9a8e51a300fbc015539cb" COMMAND_TIMEOUT_SECONDS = 30 CHILD_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" RUN_ID_RE = re.compile(r"[a-z0-9][a-z0-9-]{6,62}[a-z0-9]\Z") @@ -195,8 +200,14 @@ CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( "column_dml_grants", "database_create_grants", "missing_allowed_table_selects", + "large_object_acl_privileges", + "large_object_mutation_routine_execute", + "public_large_object_mutation_routine_execute", + "other_scoped_backends", "owned_database_objects", + "owned_large_objects", "owner_database_create_grants", + "owner_database_connect_grants", "owner_direct_database_acl_entries", "owner_missing_agent_select_columns", "owner_missing_proposal_insert_columns", @@ -210,38 +221,44 @@ CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( "owner_unexpected_schema_usage", "owner_unexpected_table_selects", "schema_create_grants", + "parameter_privileges", + "public_database_connect_grants", + "scoped_prepared_xacts", "sequence_grants", "table_dml_grants", "unexpected_column_selects", "unexpected_direct_database_acl_entries", + "unexpected_connectable_databases", "unexpected_routine_execute", + "unexpected_role_settings", "unexpected_schema_usage", "unexpected_security_definer_execute", "unexpected_table_selects", + "unsafe_allowed_relation_kinds", + "unsafe_allowed_relation_inheritance", + "unsafe_proposal_column_contract", + "unsafe_proposal_default_dependencies", + "unsafe_proposal_defaults", + "unsafe_proposal_constraint_dependencies", + "unsafe_proposal_index_expressions", + "unsafe_proposal_policies", + "unsafe_proposal_rewrite_rules", + "unsafe_proposal_triggers", + "unsafe_proposal_generated_columns", ) CATALOG_TRUE_FIELDS: tuple[str, ...] = ( "canonical_connect", "owner_grant_contract", "owner_schema_usage", + "proposal_table_contract", + "proposal_constraint_shape", "runtime_connect_acl_exact", "runtime_schema_usage", + "runtime_setting_contract", + "effective_setting_contract", ) -LEGACY_CATALOG_ZERO_COUNT_FIELDS: tuple[str, ...] = ( - "owner_column_privileges", - "owner_routine_privileges", - "owner_schema_privileges", - "owner_sequence_privileges", - "owner_table_privileges", - "runtime_column_privileges", - "runtime_routine_privileges", - "runtime_schema_privileges", - "runtime_sequence_privileges", - "runtime_table_privileges", -) - - @dataclass(frozen=True) class CommandResult: returncode: int @@ -250,6 +267,7 @@ class CommandResult: Runner = Callable[[list[str], Mapping[str, str], int, bool], CommandResult] +DependencyValidator = Callable[[], None] @dataclass(frozen=True) @@ -258,6 +276,7 @@ class NegativeCheck: database: str sql: str expected_sqlstate: str + expected_connection_denial: bool = False class VerificationError(RuntimeError): @@ -328,14 +347,110 @@ def _base_child_environment(_source: Mapping[str, str]) -> dict[str, str]: "LANG": "C", "LC_ALL": "C", "PATH": CHILD_PATH, + "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", } +def _path_and_parents(path: Path) -> tuple[Path, ...]: + resolved = path if path.is_absolute() else path.absolute() + return (resolved, *resolved.parents) + + +def _assert_root_owned_nonwritable(path: Path, *, follow_symlinks: bool = True) -> None: + try: + metadata = path.stat() if follow_symlinks else path.lstat() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if metadata.st_uid != 0 or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + if os.geteuid() != 0 and os.access(path, os.W_OK): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + + +def _assert_trusted_executable(raw_path: str) -> None: + requested = Path(raw_path) + if not requested.is_absolute(): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + try: + resolved = requested.resolve(strict=True) + metadata = resolved.stat() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if not stat.S_ISREG(metadata.st_mode) or not os.access(resolved, os.X_OK): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + try: + requested_metadata = requested.lstat() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if requested_metadata.st_uid != 0 or ( + not stat.S_ISLNK(requested_metadata.st_mode) + and requested_metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + ): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + for candidate in (*_path_and_parents(requested.parent), *_path_and_parents(resolved)): + _assert_root_owned_nonwritable(candidate) + + +def validate_runtime_dependencies() -> None: + _assert_trusted_executable(GCLOUD_BIN) + _assert_trusted_executable(PSQL_BIN) + runtime_cloudsdk_config_path() + server_ca = runtime_server_ca_path() + for candidate in _path_and_parents(server_ca): + _assert_root_owned_nonwritable(candidate) + try: + certificate = server_ca.read_bytes() + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if hashlib.sha256(certificate).hexdigest() != SERVER_CA_SHA256: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + + +def runtime_server_ca_path() -> Path: + raw = os.environ.get("TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT", str(SERVER_CA_PATH)) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + if resolved != candidate or not resolved.is_file(): + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") + return resolved + + +def selected_server_ca_path() -> Path: + return Path(os.environ.get("TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT", str(SERVER_CA_PATH))) + + +def runtime_cloudsdk_config_path() -> Path: + raw = os.environ.get("TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG", str(RUNTIME_CLOUDSDK_CONFIG)) + candidate = Path(raw) + try: + if not candidate.is_absolute() or candidate.is_symlink(): + raise OSError + resolved = candidate.resolve(strict=True) + if resolved != candidate or not resolved.is_dir() or any(resolved.iterdir()): + raise OSError + except OSError: + raise VerificationError("runtime_dependency_untrusted", "runtime_dependencies") from None + for part in _path_and_parents(resolved): + _assert_root_owned_nonwritable(part) + return resolved + + +def selected_cloudsdk_config_path() -> Path: + return Path(os.environ.get("TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG", str(RUNTIME_CLOUDSDK_CONFIG))) + + def _gcloud_environment(source: Mapping[str, str], config_dir: Path) -> dict[str, str]: env = _base_child_environment(source) env.update( { "CLOUDSDK_CONFIG": str(config_dir), + "CLOUDSDK_CORE_DISABLE_FILE_LOGGING": "true", "CLOUDSDK_CORE_DISABLE_PROMPTS": "1", "CLOUDSDK_CORE_PROJECT": PROJECT_ID, } @@ -345,7 +460,13 @@ def _gcloud_environment(source: Mapping[str, str], config_dir: Path) -> dict[str def _psql_environment(source: Mapping[str, str], password: str) -> dict[str, str]: env = _base_child_environment(source) - env["PGPASSWORD"] = password + env.update( + { + "PGPASSWORD": password, + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": str(selected_server_ca_path()), + } + ) return env @@ -424,7 +545,7 @@ def _assert_administrator_secret_denied(*, env: Mapping[str, str], runner: Runne def _conninfo(database: str) -> str: return ( f"host={PRIVATE_CLOUDSQL_HOST} port={PRIVATE_CLOUDSQL_PORT} dbname={database} " - f"user={RUNTIME_DATABASE_ROLE} sslmode=require connect_timeout=10 " + f"user={RUNTIME_DATABASE_ROLE} sslmode=verify-ca sslrootcert={selected_server_ca_path()} connect_timeout=10 " "application_name=leoclean_runtime_permission_verifier" ) @@ -515,7 +636,8 @@ select pg_catalog.jsonb_build_object( 'ssl_version', coalesce( (select version from pg_catalog.pg_stat_ssl where pid = pg_catalog.pg_backend_pid()), '' - ) + ), + 'system_identifier', (select system_identifier::text from pg_catalog.pg_control_system()) )::text; """.strip() @@ -689,17 +811,60 @@ with runtime_role as ( ), allowed_relation as ( select allowed_select.nspname, allowed_select.relname, - relation.oid + relation.oid, + relation.relkind from allowed_select left join pg_catalog.pg_namespace namespace on namespace.nspname = allowed_select.nspname left join pg_catalog.pg_class relation on relation.relnamespace = namespace.oid and relation.relname = allowed_select.relname - and relation.relkind in ('r', 'p', 'v', 'm', 'f') ), owner_insert_column(attname) as ( values {owner_insert_values} +), expected_proposal_column(ordinal, attname, typname, not_null, has_default, collated) as ( + values + (1, 'id', 'uuid', true, true, false), + (2, 'proposal_type', 'text', true, false, true), + (3, 'status', 'text', true, true, true), + (4, 'proposed_by_handle', 'text', false, false, true), + (5, 'proposed_by_agent_id', 'uuid', false, false, false), + (6, 'channel', 'text', true, true, true), + (7, 'source_ref', 'text', false, false, true), + (8, 'rationale', 'text', true, false, true), + (9, 'payload', 'jsonb', true, false, false), + (10, 'reviewed_by_handle', 'text', false, false, true), + (11, 'reviewed_by_agent_id', 'uuid', false, false, false), + (12, 'reviewed_at', 'timestamptz', false, false, false), + (13, 'review_note', 'text', false, false, true), + (14, 'applied_by_handle', 'text', false, false, true), + (15, 'applied_by_agent_id', 'uuid', false, false, false), + (16, 'applied_at', 'timestamptz', false, false, false), + (17, 'created_at', 'timestamptz', true, true, false), + (18, 'updated_at', 'timestamptz', true, true, false) +), proposal_table as ( + select relation.* + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + where namespace.nspname = 'kb_stage' + and relation.relname = 'kb_proposals' +), expected_proposal_default(attname, expression) as ( + values + ('id', 'gen_random_uuid()'), + ('status', '''pending_review''::text'), + ('channel', '''cli''::text'), + ('created_at', 'now()'), + ('updated_at', 'now()') +), actual_proposal_default as ( + select attribute.attname, + pg_catalog.pg_get_expr(default_row.adbin, default_row.adrelid) as expression + from proposal_table + join pg_catalog.pg_attribute attribute on attribute.attrelid = proposal_table.oid + join pg_catalog.pg_attrdef default_row + on default_row.adrelid = attribute.attrelid + and default_row.adnum = attribute.attnum + where attribute.attnum > 0 + and not attribute.attisdropped ) select pg_catalog.jsonb_build_object( 'canonical_connect', coalesce( @@ -720,6 +885,75 @@ select pg_catalog.jsonb_build_object( where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) and acl.grantee = runtime_role.oid ), false), + 'runtime_setting_contract', coalesce(( + select pg_catalog.count(*) = 1 + and pg_catalog.bool_and( + setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ) + from pg_catalog.pg_db_role_setting setting_row + join runtime_role on runtime_role.oid = setting_row.setrole + ), false), + 'unexpected_role_settings', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_db_role_setting setting_row + where setting_row.setrole in ((select oid from runtime_role), (select oid from owner_role)) + and not ( + setting_row.setrole = (select oid from runtime_role) + and setting_row.setdatabase = ( + select database_row.oid + from pg_catalog.pg_database database_row + where database_row.datname = {_sql_literal(CANONICAL_DATABASE)} + ) + and pg_catalog.cardinality(setting_row.setconfig) = 3 + and setting_row.setconfig @> array[ + 'search_path=pg_catalog, public, kb_stage', + 'statement_timeout=15s', + 'lock_timeout=2s' + ]::text[] + ) + ), + 'effective_setting_contract', ( + pg_catalog.current_setting('search_path') = 'pg_catalog, public, kb_stage' + and pg_catalog.current_setting('statement_timeout') = '15s' + and pg_catalog.current_setting('lock_timeout') = '2s' + and pg_catalog.current_setting('session_preload_libraries') = '' + and pg_catalog.current_setting('local_preload_libraries') = '' + and pg_catalog.current_setting('role') = 'none' + ), + 'unexpected_connectable_databases', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + where database_row.datname <> {_sql_literal(CANONICAL_DATABASE)} + and pg_catalog.has_database_privilege(current_user, database_row.oid, 'CONNECT') + ), + 'owner_database_connect_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + where pg_catalog.has_database_privilege( + {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, + database_row.oid, + 'CONNECT' + ) + ), + 'public_database_connect_grants', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_database database_row + cross join lateral pg_catalog.aclexplode( + coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) + ) acl + where acl.grantee = 0 + and acl.privilege_type = 'CONNECT' + ), 'unexpected_direct_database_acl_entries', ( select pg_catalog.count(*)::int from pg_catalog.pg_database database_row @@ -727,8 +961,7 @@ select pg_catalog.jsonb_build_object( cross join lateral pg_catalog.aclexplode( coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) ) acl - where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) - and acl.grantee = runtime_role.oid + where acl.grantee = runtime_role.oid and not ( database_row.datname = {_sql_literal(CANONICAL_DATABASE)} and acl.privilege_type = 'CONNECT' @@ -742,20 +975,19 @@ select pg_catalog.jsonb_build_object( cross join lateral pg_catalog.aclexplode( coalesce(database_row.datacl, pg_catalog.acldefault('d', database_row.datdba)) ) acl - where database_row.datname in ({_sql_literal(CANONICAL_DATABASE)}, {_sql_literal(LEGACY_DATABASE)}) - and acl.grantee = owner_role.oid + where acl.grantee = owner_role.oid ), 'database_create_grants', ( select pg_catalog.count(*)::int - from (values ({_sql_literal(CANONICAL_DATABASE)}), ({_sql_literal(LEGACY_DATABASE)})) database_name(name) - where pg_catalog.has_database_privilege(current_user, database_name.name, 'CREATE') + from pg_catalog.pg_database database_row + where pg_catalog.has_database_privilege(current_user, database_row.oid, 'CREATE') ), 'owner_database_create_grants', ( select pg_catalog.count(*)::int - from (values ({_sql_literal(CANONICAL_DATABASE)}), ({_sql_literal(LEGACY_DATABASE)})) database_name(name) + from pg_catalog.pg_database database_row where pg_catalog.has_database_privilege( {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, - database_name.name, + database_row.oid, 'CREATE' ) ), @@ -837,6 +1069,241 @@ select pg_catalog.jsonb_build_object( false ) ), + 'unsafe_allowed_relation_kinds', ( + select pg_catalog.count(*)::int + from allowed_relation + where allowed_relation.relkind is distinct from 'r'::"char" + ), + 'unsafe_allowed_relation_inheritance', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_inherits inheritance + where inheritance.inhparent in (select oid from allowed_relation where oid is not null) + or inheritance.inhrelid in (select oid from allowed_relation where oid is not null) + ), + 'unsafe_proposal_column_contract', ( + select pg_catalog.count(*)::int + from expected_proposal_column expected + full join ( + select attribute.attnum::int as ordinal, + attribute.attname, + type_row.typname, + attribute.attnotnull as not_null, + attribute.atthasdef as has_default, + attribute.attidentity, + attribute.attgenerated, + attribute.atttypmod, + type_namespace.nspname as type_namespace, + attribute.attcollation = 'pg_catalog.default'::pg_catalog.regcollation as collated + from proposal_table + join pg_catalog.pg_attribute attribute on attribute.attrelid = proposal_table.oid + join pg_catalog.pg_type type_row on type_row.oid = attribute.atttypid + join pg_catalog.pg_namespace type_namespace on type_namespace.oid = type_row.typnamespace + where attribute.attnum > 0 + and not attribute.attisdropped + ) actual using (ordinal, attname) + where expected.ordinal is null + or actual.ordinal is null + or expected.typname is distinct from actual.typname + or expected.not_null is distinct from actual.not_null + or expected.has_default is distinct from actual.has_default + or expected.collated is distinct from actual.collated + or actual.attidentity <> '' + or actual.attgenerated <> '' + or actual.atttypmod <> -1 + or actual.type_namespace <> 'pg_catalog' + ), + 'proposal_table_contract', coalesce(( + select pg_catalog.count(*) = 1 + and pg_catalog.bool_and( + proposal_table.relkind = 'r' + and proposal_table.relpersistence = 'p' + and not proposal_table.relrowsecurity + and not proposal_table.relforcerowsecurity + ) + from proposal_table + ), false), + 'unsafe_proposal_triggers', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_trigger trigger_row + join proposal_table on proposal_table.oid = trigger_row.tgrelid + where not trigger_row.tgisinternal + ), + 'unsafe_proposal_rewrite_rules', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_rewrite rewrite_row + join proposal_table on proposal_table.oid = rewrite_row.ev_class + ), + 'unsafe_proposal_policies', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_policy policy_row + join proposal_table on proposal_table.oid = policy_row.polrelid + ), + 'unsafe_proposal_generated_columns', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_attribute attribute + join proposal_table on proposal_table.oid = attribute.attrelid + where attribute.attnum > 0 + and not attribute.attisdropped + and attribute.attgenerated <> '' + ), + 'unsafe_proposal_defaults', ( + select pg_catalog.count(*)::int + from expected_proposal_default expected + full join actual_proposal_default actual using (attname) + where expected.expression is distinct from actual.expression + ), + 'unsafe_proposal_default_dependencies', ( + select pg_catalog.count(*)::int + from ( + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + join pg_catalog.pg_proc function_row on function_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname <> 'pg_catalog' + or function_row.oid not in ( + 'pg_catalog.gen_random_uuid()'::pg_catalog.regprocedure, + 'pg_catalog.now()'::pg_catalog.regprocedure + ) + union all + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_type'::pg_catalog.regclass + join pg_catalog.pg_type type_row on type_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = type_row.typnamespace + where namespace.nspname <> 'pg_catalog' + union all + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_operator'::pg_catalog.regclass + join pg_catalog.pg_operator operator_row on operator_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = operator_row.oprnamespace + where namespace.nspname <> 'pg_catalog' + union all + select dependency.objid, dependency.refobjid + from proposal_table + join pg_catalog.pg_attrdef default_row on default_row.adrelid = proposal_table.oid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass + and dependency.objid = default_row.oid + and dependency.refclassid = 'pg_catalog.pg_collation'::pg_catalog.regclass + join pg_catalog.pg_collation collation_row on collation_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = collation_row.collnamespace + where namespace.nspname <> 'pg_catalog' + ) unsafe_dependency + ), + 'owned_large_objects', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_largeobject_metadata metadata + where metadata.lomowner in ((select oid from runtime_role), (select oid from owner_role)) + ), + 'large_object_acl_privileges', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_largeobject_metadata metadata + where exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(metadata.lomacl, pg_catalog.acldefault('L', metadata.lomowner)) + ) acl + where acl.grantee in (0, (select oid from runtime_role), (select oid from owner_role)) + and acl.privilege_type in ('SELECT', 'UPDATE') + ) + ), + 'large_object_mutation_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + cross join (values ({_sql_literal(RUNTIME_DATABASE_ROLE)}), ({_sql_literal(STAGE_OWNER_DATABASE_ROLE)})) scoped_role(name) + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and pg_catalog.has_function_privilege(scoped_role.name, function_row.oid, 'EXECUTE') + ), + 'public_large_object_mutation_routine_execute', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_proc function_row + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + cross join lateral pg_catalog.aclexplode( + coalesce(function_row.proacl, pg_catalog.acldefault('f', function_row.proowner)) + ) acl + where namespace.nspname = 'pg_catalog' + and function_row.proname in ( + 'lo_creat', 'lo_create', 'lo_export', 'lo_from_bytea', 'lo_import', + 'lo_open', 'lo_put', 'lo_truncate', 'lo_truncate64', 'lo_unlink', 'lowrite' + ) + and acl.grantee = 0 + and acl.privilege_type = 'EXECUTE' + ), + 'parameter_privileges', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_parameter_acl parameter_acl + cross join (values ({_sql_literal(RUNTIME_DATABASE_ROLE)}), ({_sql_literal(STAGE_OWNER_DATABASE_ROLE)})) scoped_role(name) + where pg_catalog.has_parameter_privilege(scoped_role.name, parameter_acl.parname, 'SET') + or pg_catalog.has_parameter_privilege(scoped_role.name, parameter_acl.parname, 'ALTER SYSTEM') + ), + 'proposal_constraint_shape', coalesce(( + select pg_catalog.count(*) = 6 + and pg_catalog.count(*) filter ( + where (constraint_row.conname, constraint_row.contype) in ( + ('kb_proposals_pkey', 'p'), + ('kb_proposals_proposal_type_check', 'c'), + ('kb_proposals_status_check', 'c'), + ('kb_proposals_applied_by_agent_id_fkey', 'f'), + ('kb_proposals_proposed_by_agent_id_fkey', 'f'), + ('kb_proposals_reviewed_by_agent_id_fkey', 'f') + ) + ) = 6 + from pg_catalog.pg_constraint constraint_row + join proposal_table on proposal_table.oid = constraint_row.conrelid + ), false), + 'unsafe_proposal_constraint_dependencies', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_constraint constraint_row + join proposal_table on proposal_table.oid = constraint_row.conrelid + join pg_catalog.pg_depend dependency + on dependency.classid = 'pg_catalog.pg_constraint'::pg_catalog.regclass + and dependency.objid = constraint_row.oid + and dependency.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass + join pg_catalog.pg_proc function_row on function_row.oid = dependency.refobjid + join pg_catalog.pg_namespace namespace on namespace.oid = function_row.pronamespace + where namespace.nspname <> 'pg_catalog' + ), + 'unsafe_proposal_index_expressions', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_index index_row + join proposal_table on proposal_table.oid = index_row.indrelid + where index_row.indexprs is not null + or ( + index_row.indpred is not null + and pg_catalog.pg_get_expr(index_row.indpred, index_row.indrelid) + <> '(proposed_by_handle IS NOT NULL)' + ) + ), + 'other_scoped_backends', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_stat_activity activity + where activity.pid <> pg_catalog.pg_backend_pid() + and activity.usename in ({_sql_literal(RUNTIME_DATABASE_ROLE)}, {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}) + ), + 'scoped_prepared_xacts', ( + select pg_catalog.count(*)::int + from pg_catalog.pg_prepared_xacts prepared + where prepared.owner in ({_sql_literal(RUNTIME_DATABASE_ROLE)}, {_sql_literal(STAGE_OWNER_DATABASE_ROLE)}) + ), 'table_dml_grants', ( select pg_catalog.count(*)::int from pg_catalog.pg_class relation @@ -1083,99 +1550,6 @@ select pg_catalog.jsonb_build_object( """.strip() -def _legacy_catalog_privilege_posture_sql() -> str: - return f""" -with scoped_role(role_name, field_prefix) as ( - values - ({_sql_literal(RUNTIME_DATABASE_ROLE)}, 'runtime'), - ({_sql_literal(STAGE_OWNER_DATABASE_ROLE)}, 'owner') -), restore_schema as ( - select oid - from pg_catalog.pg_namespace - where nspname = 'teleo_restore' -), posture as ( - select scoped_role.field_prefix, - ( - select pg_catalog.count(*)::int - from restore_schema - cross join (values ('USAGE'), ('CREATE')) privilege(name) - where pg_catalog.has_schema_privilege( - scoped_role.role_name, - restore_schema.oid, - privilege.name - ) - ) as schema_privileges, - ( - select pg_catalog.count(*)::int - from pg_catalog.pg_class relation - join restore_schema on restore_schema.oid = relation.relnamespace - cross join ( - values ('SELECT'), ('INSERT'), ('UPDATE'), ('DELETE'), - ('TRUNCATE'), ('REFERENCES'), ('TRIGGER') - ) privilege(name) - where relation.relkind in ('r', 'p', 'v', 'm', 'f') - and pg_catalog.has_table_privilege( - scoped_role.role_name, - relation.oid, - privilege.name - ) - ) as table_privileges, - ( - select pg_catalog.count(*)::int - from pg_catalog.pg_class relation - join restore_schema on restore_schema.oid = relation.relnamespace - join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid - cross join (values ('SELECT'), ('INSERT'), ('UPDATE'), ('REFERENCES')) privilege(name) - where relation.relkind in ('r', 'p', 'v', 'm', 'f') - and attribute.attnum > 0 - and not attribute.attisdropped - and pg_catalog.has_column_privilege( - scoped_role.role_name, - relation.oid, - attribute.attnum, - privilege.name - ) - ) as column_privileges, - ( - select pg_catalog.count(*)::int - from pg_catalog.pg_class relation - join restore_schema on restore_schema.oid = relation.relnamespace - cross join (values ('USAGE'), ('SELECT'), ('UPDATE')) privilege(name) - where relation.relkind = 'S' - and pg_catalog.has_sequence_privilege( - scoped_role.role_name, - relation.oid, - privilege.name - ) - ) as sequence_privileges, - ( - select pg_catalog.count(*)::int - from pg_catalog.pg_proc function_row - join restore_schema on restore_schema.oid = function_row.pronamespace - where pg_catalog.has_function_privilege( - scoped_role.role_name, - function_row.oid, - 'EXECUTE' - ) - ) as routine_privileges - from scoped_role -) -select pg_catalog.jsonb_build_object( - 'runtime_schema_privileges', max(schema_privileges) filter (where field_prefix = 'runtime'), - 'runtime_table_privileges', max(table_privileges) filter (where field_prefix = 'runtime'), - 'runtime_column_privileges', max(column_privileges) filter (where field_prefix = 'runtime'), - 'runtime_sequence_privileges', max(sequence_privileges) filter (where field_prefix = 'runtime'), - 'runtime_routine_privileges', max(routine_privileges) filter (where field_prefix = 'runtime'), - 'owner_schema_privileges', max(schema_privileges) filter (where field_prefix = 'owner'), - 'owner_table_privileges', max(table_privileges) filter (where field_prefix = 'owner'), - 'owner_column_privileges', max(column_privileges) filter (where field_prefix = 'owner'), - 'owner_sequence_privileges', max(sequence_privileges) filter (where field_prefix = 'owner'), - 'owner_routine_privileges', max(routine_privileges) filter (where field_prefix = 'owner') -)::text -from posture; -""".strip() - - def _canary_count_sql(source_ref: str) -> str: return f"select pg_catalog.count(*)::text from kb_stage.kb_proposals where source_ref = {_sql_literal(source_ref)};" @@ -1202,6 +1576,7 @@ select pg_catalog.jsonb_build_object( 'proposal', proposal )::text from staged; +set constraints all immediate; rollback; """.strip() @@ -1249,6 +1624,24 @@ def _negative_checks(run_id: str, source_ref: str) -> tuple[NegativeCheck, ...]: _transactional("delete from public.claims where false"), "42501", ), + NegativeCheck( + "large_object_creat", + CANONICAL_DATABASE, + _transactional("select pg_catalog.lo_creat(0)"), + "42501", + ), + NegativeCheck( + "large_object_create", + CANONICAL_DATABASE, + _transactional("select pg_catalog.lo_create(0)"), + "42501", + ), + NegativeCheck( + "large_object_from_bytea", + CANONICAL_DATABASE, + _transactional("select pg_catalog.lo_from_bytea(0, ''::bytea)"), + "42501", + ), NegativeCheck( "forged_proposer_overload", CANONICAL_DATABASE, @@ -1313,10 +1706,18 @@ def _negative_checks(run_id: str, source_ref: str) -> tuple[NegativeCheck, ...]: "42501", ), NegativeCheck( - "legacy_teleo_restore_read", + "legacy_database_connect", LEGACY_DATABASE, - _transactional("select 1 from teleo_restore.response_audit limit 1"), + "select 1 /* legacy_database_connect */;", "42501", + expected_connection_denial=True, + ), + NegativeCheck( + "template_database_connect", + TEMPLATE_DATABASE, + "select 1 /* template_database_connect */;", + "42501", + expected_connection_denial=True, ), ) @@ -1341,6 +1742,24 @@ def _expect_sqlstate( observed_sqlstate="success", ) + if check.expected_connection_denial: + normalized = result.stderr.casefold() + expected_database = f'permission denied for database "{check.database}"'.casefold() + if expected_database not in normalized or "does not have connect privilege" not in normalized: + raise VerificationError( + "negative_connection_denial_mismatch", + check.name, + expected_sqlstate=check.expected_sqlstate, + observed_sqlstate="startup_error_unclassified", + ) + return { + "check": check.name, + "expected_sqlstate": check.expected_sqlstate, + "observed_error_class": "connect_privilege_denied", + "observed_sqlstate": "not_exposed_by_libpq_startup", + "result": "denied", + } + observed = SQLSTATE_RE.findall(result.stderr) unique_observed = sorted(set(observed)) if not observed: @@ -1373,6 +1792,7 @@ def _assert_identity(identity: dict[str, Any]) -> dict[str, Any]: "server_addr": PRIVATE_CLOUDSQL_HOST, "server_port": PRIVATE_CLOUDSQL_PORT, "ssl": True, + "system_identifier": EXPECTED_SYSTEM_IDENTIFIER, } if any(identity.get(key) != value for key, value in expected.items()): raise VerificationError("private_ssl_identity_mismatch", "database_identity") @@ -1510,16 +1930,6 @@ def _assert_catalog_privilege_posture(posture: dict[str, Any]) -> dict[str, int return sanitized -def _assert_legacy_catalog_privilege_posture(posture: dict[str, Any]) -> dict[str, int]: - sanitized: dict[str, int] = {} - for field in LEGACY_CATALOG_ZERO_COUNT_FIELDS: - value = posture.get(field) - if isinstance(value, bool) or not isinstance(value, int) or value != 0: - raise VerificationError("legacy_catalog_privilege_posture_mismatch", "legacy_catalog_privileges") - sanitized[field] = 0 - return sanitized - - def _assert_allowed_reads(readback: dict[str, Any]) -> dict[str, int]: sanitized: dict[str, int] = {} for key in ("claims_rows_sampled", "proposal_rows_sampled"): @@ -1553,6 +1963,7 @@ def verify_runtime_permissions( runner: Runner = subprocess_runner, base_env: Mapping[str, str] | None = None, effective_user: str | None = None, + dependency_validator: DependencyValidator = validate_runtime_dependencies, ) -> dict[str, Any]: try: run_id = validate_run_id(run_id) @@ -1566,14 +1977,13 @@ def verify_runtime_permissions( raise VerificationError("unix_user_unknown", "runtime_user") from None if effective_user != RUNTIME_UNIX_USER: raise VerificationError("unexpected_unix_user", "runtime_user") + dependency_validator() source = os.environ if base_env is None else base_env source_ref = f"leo-runtime-permission:{run_id}" generated_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") - with tempfile.TemporaryDirectory(prefix="leoclean-runtime-permission-gcloud-") as config_name: - config_dir = Path(config_name) - config_dir.chmod(0o700) + with nullcontext(selected_cloudsdk_config_path()) as config_dir: gcloud_env = _gcloud_environment(source, config_dir) password = _read_scoped_password(env=gcloud_env, runner=runner) psql_env = _psql_environment(source, password) @@ -1650,18 +2060,6 @@ def verify_runtime_permissions( "catalog_privilege_posture", ) ) - legacy_catalog_privileges = _assert_legacy_catalog_privilege_posture( - _parse_json_object( - _run_psql_success( - check="legacy_catalog_privileges", - database=LEGACY_DATABASE, - sql=_legacy_catalog_privilege_posture_sql(), - env=psql_env, - runner=runner, - ), - "legacy_catalog_privileges", - ) - ) allowed_reads = _assert_allowed_reads( _parse_json_object( _run_psql_success( @@ -1721,7 +2119,6 @@ def verify_runtime_permissions( "canary_rows_after": rows_after, "canary_rows_before": rows_before, "function_privileges": function_privileges, - "legacy_catalog_privileges": legacy_catalog_privileges, "negative_permissions": negative_permissions, "role_posture": role_posture, "stage_function_definition": stage_function_definition, @@ -1768,7 +2165,7 @@ def verify_runtime_permissions( "port": PRIVATE_CLOUDSQL_PORT, "project": PROJECT_ID, "role": RUNTIME_DATABASE_ROLE, - "sslmode": "require", + "sslmode": "verify-ca", }, } diff --git a/ops/verify_gcp_leoclean_service_environment.py b/ops/verify_gcp_leoclean_service_environment.py index 8dff5a1..d278d7f 100755 --- a/ops/verify_gcp_leoclean_service_environment.py +++ b/ops/verify_gcp_leoclean_service_environment.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 -I """Verify the effective Leo service environment without exposing its values.""" from __future__ import annotations @@ -18,6 +18,9 @@ RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime" SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password" RUNTIME_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +RUNTIME_CLOUDSDK_CONFIG = "/usr/local/libexec/livingip/leoclean-kb/gcloud-config" +RUNTIME_SSL_MODE = "verify-ca" +RUNTIME_SSL_ROOT_CERT = "/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem" FORBIDDEN_EXACT_FIELDS = frozenset( { @@ -26,6 +29,7 @@ FORBIDDEN_EXACT_FIELDS = frozenset( "BASHOPTS", "CDPATH", "CURL_CA_BUNDLE", + "CREDENTIALS_DIRECTORY", "ENV", "GLOBIGNORE", "GOOGLE_APPLICATION_CREDENTIALS", @@ -33,12 +37,22 @@ FORBIDDEN_EXACT_FIELDS = frozenset( "HTTP_PROXY", "IFS", "NO_PROXY", + "OPENSSL_CONF", + "OPENSSL_ENGINES", + "OPENSSL_MODULES", "PS4", "REQUESTS_CA_BUNDLE", "SHELLOPTS", "SSL_CERT_DIR", "SSL_CERT_FILE", "SSLKEYLOGFILE", + "GCONV_PATH", + "TELEO_CLOUDSQL_CREDENTIAL_MODE", + "TELEO_KB_CLAIM_BASE_URL", + "TELEO_KB_READ_ATTEMPTS", + "TELEO_MEMORY_INCLUDE_EXCERPTS", + "TELEO_MEMORY_REDACT", + "XDG_CONFIG_HOME", } ) @@ -83,7 +97,12 @@ def expected_environment() -> dict[str, str]: """Build the exact reviewed environment contract.""" return { + "CLOUDSDK_CONFIG": RUNTIME_CLOUDSDK_CONFIG, "PATH": RUNTIME_PATH, + "PGSSLMODE": RUNTIME_SSL_MODE, + "PGSSLROOTCERT": RUNTIME_SSL_ROOT_CERT, + "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", "TELEO_CANONICAL_CLOUDSQL_DB": CANONICAL_DATABASE, "TELEO_CLOUDSQL_DB": CANONICAL_DATABASE, "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK": "0", @@ -127,22 +146,31 @@ def _forbidden_field_labels(environment: Mapping[str, str]) -> tuple[str, ...]: labels: set[str] = set() for key in environment: normalized = key.upper() - if normalized.startswith("PG"): + if normalized.startswith("PG") and normalized not in {"PGSSLMODE", "PGSSLROOTCERT"}: labels.add("PG*") - if normalized.startswith("CLOUDSDK_"): + if normalized.startswith("CLOUDSDK_") and normalized != "CLOUDSDK_CONFIG": labels.add("CLOUDSDK_*") if normalized.startswith("LD_"): labels.add("LD_*") if normalized.startswith("BASH_FUNC_"): labels.add("BASH_FUNC_*") - if normalized.startswith("PYTHON") and normalized != "PYTHONUNBUFFERED": + if normalized.startswith("PYTHON") and normalized not in { + "PYTHONNOUSERSITE", + "PYTHONSAFEPATH", + "PYTHONUNBUFFERED", + }: labels.add("PYTHON*") if normalized in FORBIDDEN_EXACT_FIELDS: labels.add(normalized) return tuple(sorted(labels)) -def validate_environment(environment: Mapping[str, str]) -> tuple[str, ...]: +def validate_environment( + environment: Mapping[str, str], + *, + allow_missing_cloudsdk_config: bool = False, + allow_missing_runtime_security_fields: bool = False, +) -> tuple[str, ...]: """Validate the effective environment and return only reviewed field names.""" if not isinstance(environment, Mapping) or any( @@ -151,6 +179,18 @@ def validate_environment(environment: Mapping[str, str]) -> tuple[str, ...]: raise VerificationError("environment_malformed") expected = expected_environment() + if allow_missing_runtime_security_fields: + for field in ( + "CLOUDSDK_CONFIG", + "PGSSLMODE", + "PGSSLROOTCERT", + "PYTHONNOUSERSITE", + "PYTHONSAFEPATH", + ): + if field not in environment: + expected.pop(field) + if allow_missing_cloudsdk_config and "CLOUDSDK_CONFIG" not in environment: + expected.pop("CLOUDSDK_CONFIG") administrator_secret = ADMINISTRATOR_PASSWORD_SECRET.casefold() if any(administrator_secret in value.casefold() for value in environment.values()): raise VerificationError("administrator_secret_reference") @@ -172,12 +212,18 @@ def verify_process_environment( pid: int, *, proc_root: Path = Path("/proc"), + allow_missing_cloudsdk_config: bool = False, + allow_missing_runtime_security_fields: bool = False, ) -> dict[str, object]: """Read and verify one process environment, returning a sanitized receipt.""" validated_pid = validate_pid(pid) raw = (proc_root / str(validated_pid) / "environ").read_bytes() - validated_fields = validate_environment(parse_environment(raw)) + validated_fields = validate_environment( + parse_environment(raw), + allow_missing_cloudsdk_config=allow_missing_cloudsdk_config, + allow_missing_runtime_security_fields=allow_missing_runtime_security_fields, + ) return { "runtime_environment": "scoped", "status": "pass", @@ -196,13 +242,19 @@ def canonical_json(payload: Mapping[str, Any]) -> str: def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = _SanitizedArgumentParser(description=__doc__, add_help=False) parser.add_argument("--pid", required=True) + parser.add_argument("--allow-missing-cloudsdk-config", action="store_true") + parser.add_argument("--allow-missing-runtime-security-fields", action="store_true") return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: try: args = parse_args(argv) - receipt = verify_process_environment(validate_pid(args.pid)) + receipt = verify_process_environment( + validate_pid(args.pid), + allow_missing_cloudsdk_config=args.allow_missing_cloudsdk_config, + allow_missing_runtime_security_fields=args.allow_missing_runtime_security_fields, + ) returncode = 0 except VerificationError as exc: receipt = failure_receipt(exc) diff --git a/scripts/run_gcp_generated_db_direct_claim_suite.py b/scripts/run_gcp_generated_db_direct_claim_suite.py index f335731..6f2cc0a 100644 --- a/scripts/run_gcp_generated_db_direct_claim_suite.py +++ b/scripts/run_gcp_generated_db_direct_claim_suite.py @@ -46,8 +46,10 @@ DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service" DEFAULT_HOST = "10.61.0.3" DEFAULT_PROJECT = "teleo-501523" DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" +DEFAULT_SSL_ROOT_CERT = Path("/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem") +DEFAULT_SYSTEM_IDENTIFIER = "7659718422914359312" DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql" -REVIEWED_CLOUDSQL_TOOL_SHA256 = "66fa84b3bd6fe67e102fbce4509ea28c3537ba5bb1091e1a1a457b7425366064" +REVIEWED_CLOUDSQL_TOOL_SHA256 = "17d5213756ea6492e4232784a04b7f584afee5dce5184b33a2227e58847472c1" REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a" COUNT_READBACK_RE = re.compile( r"DB readback:\s*claims:\s*`?(?P\d+)`?;\s*" @@ -192,10 +194,13 @@ def run_target_psql( **os.environ, "PGPASSWORD": password, "PGOPTIONS": "-c default_transaction_read_only=on", + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": str(args.ssl_root_cert), } command = [ "psql", - f"host={args.host} port=5432 dbname={args.target_db} user=postgres sslmode=require connect_timeout=8", + f"host={args.host} port=5432 dbname={args.target_db} user=postgres " + f"sslmode=verify-ca sslrootcert={args.ssl_root_cert} connect_timeout=8", "-X", "-Atq", "-v", @@ -232,7 +237,11 @@ rollback; return json.loads(rows[-1]) -def validate_database_identity(identity: dict[str, Any], target_db: str) -> None: +def validate_database_identity( + identity: dict[str, Any], + target_db: str, + expected_system_identifier: str = DEFAULT_SYSTEM_IDENTIFIER, +) -> None: if identity.get("current_database") != target_db: raise bound.CheckpointError("Cloud SQL identity does not match the generated target database") if identity.get("ssl") is not True: @@ -252,8 +261,8 @@ def validate_database_identity(identity: dict[str, Any], target_db: str) -> None ) if not any(address in network for network in private_networks): raise bound.CheckpointError("Cloud SQL target address is not RFC1918-private") - if not identity.get("system_identifier"): - raise bound.CheckpointError("Cloud SQL target system identifier is missing") + if identity.get("system_identifier") != expected_system_identifier: + raise bound.CheckpointError("Cloud SQL target system identifier is not the reviewed instance") def canonical_status(args: argparse.Namespace) -> dict[str, Any]: @@ -347,6 +356,7 @@ def build_cloudsql_wrapper( host: str, project: str, password_secret: str, + ssl_root_cert: Path, run_nonce: str, system_exec_path: str = bound.SYSTEM_EXEC_PATH, ) -> str: @@ -361,18 +371,21 @@ TARGET_HOST={shlex.quote(host)} SOURCE_COMPUTE={shlex.quote(SOURCE_COMPUTE)} PROJECT={shlex.quote(project)} PASSWORD_SECRET={shlex.quote(password_secret)} +SSL_ROOT_CERT={shlex.quote(str(ssl_root_cert))} RUN_NONCE={shlex.quote(run_nonce)} PYTHON={shlex.quote(python)} export PATH={shlex.quote(system_exec_path)} export CLOUDSDK_CONFIG=/home/teleo/.config/gcloud export PGOPTIONS="-c default_transaction_read_only=on" +export PGSSLMODE=verify-ca +export PGSSLROOTCERT="$SSL_ROOT_CERT" INVOCATION_ID="$($PYTHON -c 'import uuid; print(uuid.uuid4())')" if ! PGPASSWORD="$(gcloud secrets versions access latest --secret="$PASSWORD_SECRET" --project="$PROJECT")"; then exit 124 fi export PGPASSWORD trap 'unset PGPASSWORD PGOPTIONS' EXIT -DB_RECEIPT="$(psql "host=$TARGET_HOST port=5432 dbname=$TARGET_DATABASE user=postgres sslmode=require connect_timeout=8" -X -Atq -v ON_ERROR_STOP=1 <<'SQL' +DB_RECEIPT="$(psql "host=$TARGET_HOST port=5432 dbname=$TARGET_DATABASE user=postgres sslmode=verify-ca sslrootcert=$SSL_ROOT_CERT connect_timeout=8" -X -Atq -v ON_ERROR_STOP=1 <<'SQL' begin transaction read only; select jsonb_build_object( 'current_database', current_database(), @@ -468,6 +481,7 @@ def patch_temp_bridge(args: argparse.Namespace, temp_profile: Path) -> dict[str, project=args.project, password_secret=args.password_secret, run_nonce=run_nonce, + ssl_root_cert=args.ssl_root_cert, ) binding = f""" ## Generated GCP Database Checkpoint @@ -788,6 +802,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--host", default=DEFAULT_HOST) parser.add_argument("--project", default=DEFAULT_PROJECT) parser.add_argument("--password-secret", default=DEFAULT_SECRET) + parser.add_argument("--ssl-root-cert", default=DEFAULT_SSL_ROOT_CERT, type=Path) parser.add_argument("--service", default=DEFAULT_SERVICE) parser.add_argument("--live-profile", default=bound.LIVE_PROFILE, type=Path) parser.add_argument("--chat-id", default=bound.DEFAULT_CHAT_ID) @@ -804,6 +819,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error("--target-db must start with teleo_clone_ and cannot name a live database") if not args.cloudsql_tool.is_file(): parser.error("--cloudsql-tool must exist") + if not args.ssl_root_cert.is_file(): + parser.error("--ssl-root-cert must exist") if not args.manifest_sql.is_file(): parser.error("--manifest-sql must exist") if not args.parity_receipt.is_file(): diff --git a/scripts/run_gcp_generated_db_working_leo_suite.py b/scripts/run_gcp_generated_db_working_leo_suite.py index 81eb247..d3bc26e 100644 --- a/scripts/run_gcp_generated_db_working_leo_suite.py +++ b/scripts/run_gcp_generated_db_working_leo_suite.py @@ -452,7 +452,8 @@ select current_database() = :'expected_database' as exact_database_bound \\gset "psql", ( f"host={self.args.host} port=5432 dbname={self.args.target_db} " - f"user={role} sslmode=require connect_timeout=8" + f"user={role} sslmode=verify-ca " + f"sslrootcert={self.args.ssl_root_cert} connect_timeout=8" ), "-X", "-Atq", @@ -461,7 +462,13 @@ select current_database() = :'expected_database' as exact_database_bound \\gset "-v", f"expected_database={self.args.target_db}", ] - env = {**os.environ, "PGPASSWORD": password, "PGOPTIONS": pgoptions} + env = { + **os.environ, + "PGPASSWORD": password, + "PGOPTIONS": pgoptions, + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": str(self.args.ssl_root_cert), + } try: try: output = gcp.run( @@ -474,6 +481,8 @@ select current_database() = :'expected_database' as exact_database_bound \\gset finally: env.pop("PGPASSWORD", None) env.pop("PGOPTIONS", None) + env.pop("PGSSLMODE", None) + env.pop("PGSSLROOTCERT", None) password = "" lines = output.splitlines() receipt_lines = [line for line in lines if line.startswith(ROLE_RECEIPT_PREFIX)] @@ -692,6 +701,7 @@ def _build_gcp_read_wrapper( host=args.host, project=args.project, password_secret=args.password_secret, + ssl_root_cert=getattr(args, "ssl_root_cert", gcp.DEFAULT_SSL_ROOT_CERT), run_nonce=run_nonce, ) wrapper = wrapper.replace( @@ -699,7 +709,11 @@ def _build_gcp_read_wrapper( f"PASSWORD_SECRET={shlex.quote(args.password_secret)}\nREAD_ROLE={shlex.quote(args.read_role)}\n", 1, ) - wrapper = wrapper.replace("user=postgres sslmode=require", "user=$READ_ROLE sslmode=require", 1) + wrapper = wrapper.replace( + "user=postgres sslmode=verify-ca", + "user=$READ_ROLE sslmode=verify-ca", + 1, + ) wrapper = wrapper.replace( " 'default_transaction_read_only', current_setting('default_transaction_read_only')\n", " 'default_transaction_read_only', current_setting('default_transaction_read_only'),\n" @@ -755,6 +769,8 @@ def _cloudsql_lifecycle_wrapper( args.instance, "--password-secret", args.password_secret, + "--ssl-root-cert", + str(args.ssl_root_cert), "--read-role", args.read_role, "--stage-password-secret", @@ -1600,6 +1616,7 @@ def parse_internal_stage_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--project", required=True) parser.add_argument("--instance", required=True) parser.add_argument("--password-secret", required=True) + parser.add_argument("--ssl-root-cert", required=True, type=Path) parser.add_argument("--read-role", required=True) parser.add_argument("--stage-password-secret", required=True) parser.add_argument("--stage-role", required=True) @@ -1609,6 +1626,8 @@ def parse_internal_stage_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--target-identity-receipt", required=True, type=Path) parser.add_argument("--target-identity-sha256", required=True) args = parser.parse_args(argv) + if not args.ssl_root_cert.is_file(): + parser.error("--ssl-root-cert must exist") args.review_role = "kb_review" args.apply_role = "kb_apply" args.allow_stage = True @@ -1680,6 +1699,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--project", default=gcp.DEFAULT_PROJECT) parser.add_argument("--instance") parser.add_argument("--password-secret", default=gcp.DEFAULT_SECRET) + parser.add_argument("--ssl-root-cert", default=gcp.DEFAULT_SSL_ROOT_CERT, type=Path) parser.add_argument("--read-role", default="kb_read") parser.add_argument("--stage-role", default=DEFAULT_STAGE_ROLE) parser.add_argument("--review-role", default=DEFAULT_REVIEW_ROLE) @@ -1713,6 +1733,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: if args.turn_timeout <= 0: parser.error("--turn-timeout must be positive") if args.execute: + if not args.ssl_root_cert.is_file(): + parser.error("--ssl-root-cert must exist for execution") receipt_inputs = ( args.target_identity_receipt, args.target_identity_sha256, diff --git a/systemd/leoclean-gcp-prod-parallel-cloudsql.conf b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf index 68d4d34..e8768fc 100644 --- a/systemd/leoclean-gcp-prod-parallel-cloudsql.conf +++ b/systemd/leoclean-gcp-prod-parallel-cloudsql.conf @@ -1,17 +1,36 @@ [Service] UnsetEnvironment=PGPASSWORD UnsetEnvironment=PGPASSFILE PGSERVICE PGHOST PGPORT PGDATABASE PGUSER PGOPTIONS -UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN +UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS CDPATH GLOBIGNORE IFS PS4 UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT LD_DEBUG LD_DEBUG_OUTPUT LD_DYNAMIC_WEAK LD_HWCAP_MASK LD_ORIGIN_PATH LD_PROFILE LD_SHOW_AUXV LD_TRACE_LOADED_OBJECTS LD_USE_LOAD_BIAS LD_VERBOSE LD_WARN UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT PYTHONUSERBASE PYTHONHTTPSVERIFY UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE -ReadOnlyPaths=/home/teleo/.hermes +UnsetEnvironment=OPENSSL_CONF OPENSSL_MODULES OPENSSL_ENGINES GCONV_PATH CREDENTIALS_DIRECTORY +UnsetEnvironment=XDG_CONFIG_HOME +UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT +ReadOnlyPaths= +ReadWritePaths= +InaccessiblePaths= +BindPaths= +BindReadOnlyPaths= +SupplementaryGroups= +ReadOnlyPaths=/home/teleo ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state /home/teleo/.hermes/profiles/leoclean/workspace +InaccessiblePaths=-/home/teleo/.config/gcloud -/home/teleo/.pgpass -/home/teleo/.pg_service.conf -/home/teleo/.postgresql BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin -CapabilityBoundingSet=~CAP_SYS_ADMIN +CapabilityBoundingSet= +AmbientCapabilities= +NoNewPrivileges=yes +Group=teleo +SupplementaryGroups=teleo Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +Environment=CLOUDSDK_CONFIG=/usr/local/libexec/livingip/leoclean-kb/gcloud-config +Environment=PGSSLMODE=verify-ca +Environment=PGSSLROOTCERT=/usr/local/libexec/livingip/leoclean-kb/cloudsql-server-ca.pem +Environment=PYTHONNOUSERSITE=1 +Environment=PYTHONSAFEPATH=1 Environment=TELEO_KB_MODE=cloudsql Environment=TELEO_GCP_METADATA_ONLY=1 Environment=TELEO_GCP_PROJECT=teleo-501523 diff --git a/tests/test_gcp_generated_db_direct_claim_suite.py b/tests/test_gcp_generated_db_direct_claim_suite.py index 475b8e7..08c72d3 100644 --- a/tests/test_gcp_generated_db_direct_claim_suite.py +++ b/tests/test_gcp_generated_db_direct_claim_suite.py @@ -109,6 +109,7 @@ def test_wrapper_binds_private_read_only_database_and_records_calls(tmp_path: Pa host="10.61.0.3", project="teleo-501523", password_secret="test-secret-name", + ssl_root_cert=tmp_path / "cloudsql-server-ca.pem", run_nonce="nonce", ) @@ -117,7 +118,8 @@ def test_wrapper_binds_private_read_only_database_and_records_calls(tmp_path: Pa assert '--canonical-db "$TARGET_DATABASE"' in wrapper assert "--credential-mode clone-readonly" in wrapper assert "--user postgres" in wrapper - assert "sslmode=require" in wrapper + assert "sslmode=verify-ca" in wrapper + assert "sslrootcert=$SSL_ROOT_CERT" in wrapper assert "begin transaction read only" in wrapper assert 'export PGOPTIONS="-c default_transaction_read_only=on"' in wrapper assert "default_transaction_read_only" in wrapper @@ -140,6 +142,7 @@ def test_database_identity_requires_private_tls_and_read_only_target() -> None: "default_transaction_read_only": "on", }, "teleo_clone_test", + "1234", ) with pytest.raises(RuntimeError, match="not RFC1918-private"): @@ -153,6 +156,7 @@ def test_database_identity_requires_private_tls_and_read_only_target() -> None: "default_transaction_read_only": "on", }, "teleo_clone_test", + "1234", ) @@ -184,6 +188,7 @@ def test_database_fingerprint_uses_full_normalized_manifest(monkeypatch: pytest. "host": "10.61.0.3", "target_db": "teleo_clone_test", "manifest_sql": manifest_sql, + "ssl_root_cert": tmp_path / "cloudsql-server-ca.pem", }, )() output = "\n".join( @@ -282,6 +287,7 @@ def test_generated_wrapper_executes_only_clone_bound_default_read_only_tool(tmp_ host="10.61.0.3", project="teleo-501523", password_secret="test-secret", + ssl_root_cert=tmp_path / "cloudsql-server-ca.pem", run_nonce="test-nonce", system_exec_path=f"{fake_bin}:/usr/bin:/bin", ) diff --git a/tests/test_gcp_generated_db_working_leo_suite.py b/tests/test_gcp_generated_db_working_leo_suite.py index a3e5526..4ac006c 100644 --- a/tests/test_gcp_generated_db_working_leo_suite.py +++ b/tests/test_gcp_generated_db_working_leo_suite.py @@ -131,6 +131,7 @@ def executor_args(**overrides: object) -> argparse.Namespace: "project": "teleo-501523", "instance": "teleo-pgvector-standby", "password_secret": "read-secret", + "ssl_root_cert": Path("ops/gcp-teleo-pgvector-standby-server-ca.pem").resolve(), "read_role": "kb_read", "stage_role": "kb_stage_writer", "review_role": "kb_review", @@ -229,6 +230,10 @@ def test_executor_binds_every_call_and_defaults_read_only_except_controller_phas assert len(calls) == 4 assert all("dbname=teleo_clone_working_leo" in call["command"][1] for call in calls) + assert all("sslmode=verify-ca" in call["command"][1] for call in calls) + assert all("sslrootcert=" in call["command"][1] for call in calls) + assert all(call["env"]["PGSSLMODE"] == "verify-ca" for call in calls) + assert all(call["env"]["PGSSLROOTCERT"].endswith("gcp-teleo-pgvector-standby-server-ca.pem") for call in calls) assert all("expected_database=teleo_clone_working_leo" in call["command"] for call in calls) assert calls[0]["env"]["PGOPTIONS"] == "-c default_transaction_read_only=on" assert [call["env"]["PGOPTIONS"] for call in calls[1:]] == [ @@ -535,6 +540,9 @@ def test_lifecycle_wrapper_exposes_only_read_or_exact_stage_and_never_delivery(t ) assert catalog_wrapper.count('"prompt_id": "DC-01"') == 2 assert '--user "$READ_ROLE"' in catalog_wrapper + assert "sslmode=verify-ca" in catalog_wrapper + assert "sslrootcert=$SSL_ROOT_CERT" in catalog_wrapper + assert "sslmode=require" not in catalog_wrapper def test_capability_receipt_rejects_superuser_or_wrong_phase_acl() -> None: diff --git a/tests/test_gcp_leoclean_runtime_permissions.py b/tests/test_gcp_leoclean_runtime_permissions.py index ca97995..8f594ff 100644 --- a/tests/test_gcp_leoclean_runtime_permissions.py +++ b/tests/test_gcp_leoclean_runtime_permissions.py @@ -31,7 +31,6 @@ class FakeRunner: function_posture_override: Mapping[str, Mapping[str, object]] | None = None, stage_definition_override: Mapping[str, object] | None = None, catalog_posture_override: Mapping[str, object] | None = None, - legacy_catalog_posture_override: Mapping[str, object] | None = None, ) -> None: self.sqlstate_override = sqlstate_override self.sqlstate_missing_for = sqlstate_missing_for @@ -46,7 +45,6 @@ class FakeRunner: } self.stage_definition_override = dict(stage_definition_override or {}) self.catalog_posture_override = dict(catalog_posture_override or {}) - self.legacy_catalog_posture_override = dict(legacy_catalog_posture_override or {}) self.calls: list[tuple[list[str], dict[str, str], int, bool]] = [] self.negative = {check.sql: check for check in verifier._negative_checks(RUN_ID, SOURCE_REF)} @@ -94,6 +92,7 @@ class FakeRunner: "session_user": verifier.RUNTIME_DATABASE_ROLE, "ssl": True, "ssl_version": "TLSv1.3", + "system_identifier": verifier.EXPECTED_SYSTEM_IDENTIFIER, } identity.update(self.identity_override) return verifier.CommandResult(0, json.dumps(identity), f"ignored {SECRET_VALUE}") @@ -179,13 +178,6 @@ class FakeRunner: ) catalog_posture.update(self.catalog_posture_override) return verifier.CommandResult(0, json.dumps(catalog_posture), f"ignored {SECRET_VALUE}") - if sql == verifier._legacy_catalog_privilege_posture_sql(): - legacy_posture: dict[str, object] = { - field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS - } - legacy_posture["leak"] = SECRET_VALUE - legacy_posture.update(self.legacy_catalog_posture_override) - return verifier.CommandResult(0, json.dumps(legacy_posture), f"ignored {SECRET_VALUE}") if sql == verifier._canary_count_sql(SOURCE_REF): return verifier.CommandResult(0, "0\n", f"ignored {SECRET_VALUE}") if sql == verifier._stage_sql(RUN_ID, SOURCE_REF): @@ -211,6 +203,15 @@ class FakeRunner: return verifier.CommandResult(0, SECRET_VALUE, f"unexpected success {SECRET_VALUE}") if check.name == self.sqlstate_missing_for: return verifier.CommandResult(1, SECRET_VALUE, f"denied without state {SECRET_VALUE}") + if check.expected_connection_denial: + return verifier.CommandResult( + 2, + SECRET_VALUE, + ( + f'connection failed: FATAL: permission denied for database "{check.database}"\n' + f"DETAIL: User does not have CONNECT privilege. {SECRET_VALUE}" + ), + ) observed = check.expected_sqlstate if self.sqlstate_override is not None and check.name == self.sqlstate_override[0]: observed = self.sqlstate_override[1] @@ -243,6 +244,7 @@ def run_success(runner: FakeRunner) -> dict[str, object]: runner=runner, base_env=poisoned_environment(), effective_user="teleo", + dependency_validator=lambda: None, ) @@ -253,8 +255,10 @@ def assert_child_boundaries(runner: FakeRunner) -> None: assert timeout == verifier.COMMAND_TIMEOUT_SECONDS pg_keys = sorted(key for key in env if key.upper().startswith("PG")) if command[0] == verifier.PSQL_BIN: - assert pg_keys == ["PGPASSWORD"] + assert pg_keys == ["PGPASSWORD", "PGSSLMODE", "PGSSLROOTCERT"] assert env["PGPASSWORD"] == SECRET_VALUE + assert env["PGSSLMODE"] == "verify-ca" + assert env["PGSSLROOTCERT"] == str(verifier.SERVER_CA_PATH) assert discard_stdout is False else: assert pg_keys == [] @@ -284,6 +288,7 @@ def test_live_receipt_contract_is_sanitized_and_uses_exact_child_environments() "session_user": verifier.RUNTIME_DATABASE_ROLE, "ssl": True, "ssl_version": "TLSv1.3", + "system_identifier": verifier.EXPECTED_SYSTEM_IDENTIFIER, } assert receipt["checks"]["rolled_back_proposal"] == { "agent_id_matches": True, @@ -331,9 +336,6 @@ def test_live_receipt_contract_is_sanitized_and_uses_exact_child_environments() **{field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS}, **{field: True for field in verifier.CATALOG_TRUE_FIELDS}, } - assert receipt["checks"]["legacy_catalog_privileges"] == { - field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS - } assert receipt["checks"]["stage_function_definition"] == { "acl_exact": True, "argument_modes": None, @@ -570,7 +572,6 @@ def test_catalog_queries_use_exact_regprocedure_signatures_and_both_membership_d owner_role_sql = verifier._stage_owner_role_posture_sql() stage_definition_sql = verifier._stage_function_definition_sql() catalog_sql = verifier._catalog_privilege_posture_sql() - legacy_sql = verifier._legacy_catalog_privilege_posture_sql() assert "pg_catalog.to_regprocedure(signature)" in function_sql for _name, signature, _exists, _execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS: @@ -597,12 +598,10 @@ def test_catalog_queries_use_exact_regprocedure_signatures_and_both_membership_d assert "nspname !~ '^pg_'" in catalog_sql assert "nspname <> 'information_schema'" in catalog_sql assert verifier.STAGE_OWNER_DATABASE_ROLE in catalog_sql - assert "teleo_restore" in legacy_sql - assert "pg_catalog.has_schema_privilege" in legacy_sql - assert "pg_catalog.has_table_privilege" in legacy_sql - assert "pg_catalog.has_column_privilege" in legacy_sql - assert "pg_catalog.has_sequence_privilege" in legacy_sql - assert "pg_catalog.has_function_privilege" in legacy_sql + assert "public_database_connect_grants" in catalog_sql + assert "public_large_object_mutation_routine_execute" in catalog_sql + assert "'lo_creat'" in catalog_sql + assert "'lo_open'" in catalog_sql for schema, relation in verifier.READ_TABLE_ALLOWLIST: assert schema in catalog_sql assert relation in catalog_sql @@ -630,38 +629,27 @@ def test_any_missing_required_catalog_privilege_fails_closed(field: str) -> None assert caught.value.code == "catalog_privilege_posture_mismatch" -@pytest.mark.parametrize("field", verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS) -def test_any_teleo_restore_catalog_privilege_fails_closed(field: str) -> None: - runner = FakeRunner(legacy_catalog_posture_override={field: 1}) - - with pytest.raises(verifier.VerificationError) as caught: - run_success(runner) - - assert caught.value.code == "legacy_catalog_privilege_posture_mismatch" - assert caught.value.check == "legacy_catalog_privileges" - assert SECRET_VALUE not in str(caught.value) - - -def test_teleo_restore_catalog_and_behavioral_denial_are_both_exercised_on_legacy_database() -> None: +def test_legacy_and_template_database_connect_are_both_behaviorally_denied() -> None: runner = FakeRunner() receipt = run_success(runner) - legacy_catalog_call = next( - command + denials = {check["check"]: check for check in receipt["checks"]["negative_permissions"]} + for name in ("legacy_database_connect", "template_database_connect"): + assert denials[name] == { + "check": name, + "expected_sqlstate": "42501", + "observed_error_class": "connect_privilege_denied", + "observed_sqlstate": "not_exposed_by_libpq_startup", + "result": "denied", + } + database_calls = [ + " ".join(command) for command, _env, _timeout, _discard in runner.calls - if any(part == f"--command={verifier._legacy_catalog_privilege_posture_sql()}" for part in command) - ) - assert any(f"dbname={verifier.LEGACY_DATABASE}" in part for part in legacy_catalog_call) - legacy_denial = next( - check for check in receipt["checks"]["negative_permissions"] if check["check"] == "legacy_teleo_restore_read" - ) - assert legacy_denial == { - "check": "legacy_teleo_restore_read", - "expected_sqlstate": "42501", - "observed_sqlstate": "42501", - "result": "denied", - } + if command[0] == verifier.PSQL_BIN + ] + assert any(f"dbname={verifier.LEGACY_DATABASE}" in command for command in database_calls) + assert any(f"dbname={verifier.TEMPLATE_DATABASE}" in command for command in database_calls) def test_required_sqlstate_mismatch_fails_closed_without_secret_in_error() -> None: @@ -693,6 +681,19 @@ def test_missing_verbose_sqlstate_fails_closed_without_raw_stderr() -> None: assert SECRET_VALUE not in json.dumps(error.as_dict()) +def test_unclassified_database_startup_failure_cannot_masquerade_as_connect_denial() -> None: + runner = FakeRunner(sqlstate_missing_for="legacy_database_connect") + + with pytest.raises(verifier.VerificationError) as caught: + run_success(runner) + + error = caught.value + assert error.code == "negative_connection_denial_mismatch" + assert error.check == "legacy_database_connect" + assert error.observed_sqlstate == "startup_error_unclassified" + assert SECRET_VALUE not in json.dumps(error.as_dict()) + + def test_unexpected_permission_success_fails_closed_and_transaction_sql_has_rollback() -> None: runner = FakeRunner(unexpected_success_for="proposal_update") diff --git a/tests/test_gcp_leoclean_service_environment.py b/tests/test_gcp_leoclean_service_environment.py index a1be9e6..449ed8d 100644 --- a/tests/test_gcp_leoclean_service_environment.py +++ b/tests/test_gcp_leoclean_service_environment.py @@ -78,7 +78,6 @@ def test_later_dropin_effective_override_fails_without_outputting_value( [ ("PGPASSWORD", "PG*"), ("pgservice", "PG*"), - ("CLOUDSDK_CONFIG", "CLOUDSDK_*"), ("CLOUDSDK_CORE_PROJECT", "CLOUDSDK_*"), ("CLOUDSDK_AUTH_ACCESS_TOKEN", "CLOUDSDK_*"), ("cloudsdk_auth_refresh_token", "CLOUDSDK_*"), @@ -102,6 +101,11 @@ def test_later_dropin_effective_override_fails_without_outputting_value( ("REQUESTS_CA_BUNDLE", "REQUESTS_CA_BUNDLE"), ("curl_ca_bundle", "CURL_CA_BUNDLE"), ("SSLKEYLOGFILE", "SSLKEYLOGFILE"), + ("TELEO_CLOUDSQL_CREDENTIAL_MODE", "TELEO_CLOUDSQL_CREDENTIAL_MODE"), + ("TELEO_KB_CLAIM_BASE_URL", "TELEO_KB_CLAIM_BASE_URL"), + ("TELEO_KB_READ_ATTEMPTS", "TELEO_KB_READ_ATTEMPTS"), + ("TELEO_MEMORY_INCLUDE_EXCERPTS", "TELEO_MEMORY_INCLUDE_EXCERPTS"), + ("TELEO_MEMORY_REDACT", "TELEO_MEMORY_REDACT"), ], ) def test_environment_file_broader_credentials_and_forbidden_fields_fail_closed( diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index 3a41541..90105f1 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -26,6 +26,8 @@ ROOT = Path(__file__).resolve().parents[1] BRIDGE_DIR = ROOT / "hermes-agent" / "leoclean-bin" DB_CONTEXT_PLUGIN = ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py" GCP_RUNTIME_ROLE_SQL = ROOT / "ops" / "gcp_leoclean_runtime_role.sql" +GCP_RUNTIME_WRAPPER = BRIDGE_DIR / "teleo-kb-gcp" +GCP_RUNTIME_PROVISIONER = ROOT / "ops" / "provision_gcp_leoclean_runtime_role.sh" def test_leoclean_bridge_files_are_present_and_parseable() -> None: @@ -42,6 +44,71 @@ def test_leoclean_bridge_files_are_present_and_parseable() -> None: subprocess.run(["bash", "-n", str(wrapper)], check=True) +@pytest.mark.parametrize( + "argument", + [ + "--host", + "--host=127.0.0.1", + "--port", + "--port=6543", + "--db", + "--db=postgres", + "--canonical-db", + "--canonical-db=postgres", + "--user", + "--user=postgres", + "--password-secret", + "--password-secret=administrator-secret", + "--project", + "--project=other-project", + "--credential-mode", + "--credential-mode=clone-readonly", + ], +) +def test_deployed_gcp_wrapper_executable_rejects_every_identity_override(argument: str) -> None: + injected = "sensitive-override-value" + completed = subprocess.run( + [str(GCP_RUNTIME_WRAPPER), argument, injected], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 64 + assert "identity and endpoint overrides are forbidden" in completed.stderr + assert injected not in completed.stderr + assert completed.stdout == "" + + +def test_gcp_wrapper_and_password_provisioner_are_executable_and_syntax_valid() -> None: + assert os.access(GCP_RUNTIME_WRAPPER, os.X_OK) + assert os.access(GCP_RUNTIME_PROVISIONER, os.X_OK) + subprocess.run(["bash", "-n", str(GCP_RUNTIME_WRAPPER)], check=True) + subprocess.run(["bash", "-n", str(GCP_RUNTIME_PROVISIONER)], check=True) + + wrapper = GCP_RUNTIME_WRAPPER.read_text(encoding="utf-8") + assert "exec /usr/bin/env -i" in wrapper + assert '/usr/bin/python3 -I "$CLOUDSQL_TOOL" "$@"' in wrapper + assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in wrapper + assert "PGSSLMODE=verify-ca" in wrapper + assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in wrapper + + provisioner = GCP_RUNTIME_PROVISIONER.read_text(encoding="utf-8") + capture_runtime = "runtime_password=${TELEO_LEOCLEAN_DB_PASSWORD-}" + capture_admin = "administrator_password=${PGPASSWORD-}" + clear_ambient = "unset TELEO_LEOCLEAN_DB_PASSWORD PGPASSWORD" + first_external_child = 'SCRIPT_DIR="$(cd -- "$script_directory" && pwd -P)"' + assert provisioner.index(capture_runtime) < provisioner.index(first_external_child) + assert provisioner.index(capture_admin) < provisioner.index(first_external_child) + assert provisioner.index(clear_ambient) < provisioner.index(first_external_child) + assert '"$SETSID_BIN" -- "$PSQL_BIN"' in provisioner + assert 'kill -TERM -- "-$provision_pid"' in provisioner + assert 'wait "$provision_pid"' in provisioner + assert "SERVER_CA_SHA256=80701e" in provisioner + assert "--no-password" in provisioner + assert "PGSSLMODE=verify-ca" in provisioner + + def test_cloudsql_wrapper_supports_expected_review_gated_commands() -> None: wrapper_text = (BRIDGE_DIR / "teleo-kb").read_text() cloudsql_text = (BRIDGE_DIR / "cloudsql_memory_tool.py").read_text() @@ -502,7 +569,7 @@ def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pyt monkeypatch.setattr(module.subprocess, "run", fake_run) assert module.run_psql(args, "select 1;") == "ok\n" - assert captured["command"] == ["psql", "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"] + assert captured["command"] == ["/usr/bin/psql", "-X", "-At", "-q", "-v", "ON_ERROR_STOP=1"] kwargs = captured["kwargs"] assert isinstance(kwargs, dict) env = kwargs["env"] @@ -512,7 +579,8 @@ def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pyt "PGPORT": "5432", "PGDATABASE": "teleo_canonical", "PGUSER": module.RUNTIME_DB_USER, - "PGSSLMODE": "require", + "PGSSLMODE": "verify-ca", + "PGSSLROOTCERT": module.RUNTIME_SSL_ROOT_CERT, "PGCONNECT_TIMEOUT": "8", "PGPASSWORD": "scoped-runtime-password", } @@ -728,7 +796,7 @@ def test_cloudsql_runtime_read_marker_binds_database_user_private_server_and_ssl "server_port": int(module.RUNTIME_CLOUDSQL_PORT), "ssl": True, "ssl_version": "TLSv1.3", - "system_identifier": "system-id", + "system_identifier": module.RUNTIME_SYSTEM_IDENTIFIER, "wal_lsn": "0/1", } captured: dict[str, object] = {} @@ -771,7 +839,7 @@ def test_cloudsql_runtime_read_marker_rejects_wrong_live_identity( "server_port": int(module.RUNTIME_CLOUDSQL_PORT), "ssl": True, "ssl_version": "TLSv1.3", - "system_identifier": "system-id", + "system_identifier": module.RUNTIME_SYSTEM_IDENTIFIER, "wal_lsn": "0/1", } marker[field] = value @@ -832,14 +900,16 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None: assert "stage_leoclean_proposal" in sql assert "'pending_review'" in sql assert "revoke all privileges on all tables in schema public, kb_stage" in sql - assert "can_insert_proposals_directly" in sql - assert "can_create_database_objects" in sql - assert "can_create_temp_objects" in sql - assert "can_connect_legacy_database" in sql + assert "grant insert" in sql.lower() assert "unexpectedly has database CREATE" in sql + assert "unexpectedly has schema CREATE" in sql + assert "runtime CONNECT reaches a noncanonical database" in sql + assert "PUBLIC retains database CONNECT" in sql + assert "PUBLIC retains large-object mutator EXECUTE" in sql assert "REVOKE TEMP FROM PUBLIC would be a" in sql assert "can execute unexpected SECURITY DEFINER functions" in sql - assert "revoke all privileges (%s) on table %I.%I from %I" in sql + assert "revoke connect on database %I from public" in sql + assert "revoke execute on function pg_catalog.%I(%s) from public" in sql assert re.search( r"grant\s+insert\s*\([^)]*proposal_type[^)]*payload[^)]*\)\s+" r"on\s+kb_stage\.kb_proposals\s+to\s+leoclean_kb_stage_owner", @@ -855,12 +925,13 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None: assert "grant delete" not in sql.lower() assert "create role leoclean_kb_runtime nologin" in sql.lower() disable_statement = "alter role leoclean_kb_runtime\n with nologin nosuperuser" - password_statement = "alter role leoclean_kb_runtime password" - assert sql.lower().index(disable_statement) < sql.lower().index(password_statement) - assert sql.lower().index(disable_statement) < sql.index("\\getenv runtime_password") - assert sql.index("\\getenv runtime_password") < sql.lower().index(password_statement) - assert sql.lower().index("with nologin nosuperuser") < sql.index("\\connect teleo_kb") - assert sql.index("\\connect teleo_kb") < sql.lower().rindex("with login nosuperuser") + password_statement = "\\password leoclean_kb_runtime" + assert sql.lower().index(disable_statement) < sql.index(password_statement) + assert "\\getenv runtime_password" not in sql + assert "password :'runtime_password'" not in sql.lower() + assert sql.lower().index("with nologin nosuperuser") < sql.index("\\connect teleo_canonical") + assert sql.index("\\connect teleo_canonical") < sql.lower().rindex("with login nosuperuser") + assert sql.lower().rindex("with login nosuperuser") < sql.rindex("commit;") assert "leoclean_kb_runtime final login attributes are unsafe" in sql @@ -869,6 +940,9 @@ def _load_module(path: Path): assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + if path.name == "cloudsql_memory_tool.py": + module.runtime_ssl_root_cert = lambda: module.RUNTIME_SSL_ROOT_CERT + module.runtime_cloudsdk_config = lambda: module.RUNTIME_CLOUDSDK_CONFIG return module diff --git a/tests/test_teleo_agent_systemd.py b/tests/test_teleo_agent_systemd.py index f6edaaa..1218a50 100644 --- a/tests/test_teleo_agent_systemd.py +++ b/tests/test_teleo_agent_systemd.py @@ -1,9 +1,33 @@ +import os +import resource +import signal import subprocess +import sys +import time from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[1] +def embedded_service_context_runner_source() -> str: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + marker = "remote_helpers=$(cat <<'REMOTE'\n" + start = script.index(marker) + len(marker) + end = script.index("\nREMOTE\n)", start) + helpers = script[start:end] + blocks = [part.split("\nPY", maxsplit=1)[0] for part in helpers.split("<<'PY'\n")[1:]] + return next(block for block in blocks if "def run_stopped_child(" in block) + + +def load_service_context_runner() -> dict[str, object]: + namespace: dict[str, object] = {"__name__": "embedded_service_context_runner_test"} + source = embedded_service_context_runner_source() + exec(compile(source, "", "exec"), namespace) + return namespace + + def test_teleo_agent_template_supports_optional_per_agent_env_file(): unit = (REPO_ROOT / "systemd" / "teleo-agent@.service").read_text() @@ -189,7 +213,7 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert "sha256sum" in script assert "--preflight-only" in script assert "--restart" in script - assert '/usr/bin/python3 "$tool_path" status --format json --redacted' in script + assert '/usr/bin/python3 -I "$tool_path" status --format json --redacted' in script assert "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" in script assert "169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" in script assert "ProxyHandler({})" in script @@ -204,7 +228,8 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): wrapper_probe = script.split("run_service_wrapper_status() {", maxsplit=1)[1].split( "\n}\n\nservice_fingerprint", maxsplit=1 )[0] - assert 'nsenter -t "$main_pid" -m --env' in wrapper_probe + assert 'run_in_service_cgroup "$main_pid"' in wrapper_probe + assert 'nsenter -t "$main_pid" -m -n --env' in wrapper_probe assert "/usr/bin/setpriv" in wrapper_probe assert "--reuid=teleo" in wrapper_probe assert "TELEO_CLOUDSQL_" not in wrapper_probe @@ -218,15 +243,54 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert "--property=EnvironmentFiles" in script assert "Service has an effective EnvironmentFile" in script assert "Service has a later drop-in" in script + assert "--property=PrivateNetwork" in script + assert "--property=PrivateUsers" in script + assert "NetworkNamespacePath" in script + assert "IPAddressDeny" in script + assert "IPAddressAllow" in script + assert "RestrictAddressFamilies" in script + assert "RestrictNetworkInterfaces" in script + assert "SystemCallFilter" in script + assert "AppArmorProfile" in script + assert "SELinuxContext" in script + assert '--property="$network_property"' in script assert "assert_administrator_secret_denied" in script assert '"stdout_discarded": True' in script assert '"credential_context": "systemd_main_pid_environment"' in script administrator_probe = script.split("assert_administrator_secret_denied() {", maxsplit=1)[1].split( "\n}\n\nrun_scoped_status", maxsplit=1 )[0] - assert 'nsenter -t "$main_pid" -m --env' in administrator_probe + assert 'run_in_service_cgroup "$main_pid"' in administrator_probe + assert 'nsenter -t "$main_pid" -m -n --env' in administrator_probe assert "/usr/bin/setpriv" in administrator_probe assert "env -i" not in administrator_probe + permission_probe = script.split("run_permission_verifier() {", maxsplit=1)[1].split( + "\n}\n\nassert_trusted_executable", maxsplit=1 + )[0] + assert 'if [ "$phase" = pre ]; then' in permission_probe + assert 'run_in_service_cgroup "$main_pid"' in permission_probe + assert 'nsenter -t "$main_pid" -m -n' in permission_probe + assert "/usr/bin/env -i" in permission_probe + assert 'verification_tmp="$(sudo mktemp -d /run/' in script + assert "run_in_service_cgroup() {" in script + assert "--property=ControlGroup" in script + assert 'Path(f"/proc/{pid}/cgroup")' in script + assert 'Path("/sys/fs/cgroup").resolve(strict=True)' in script + assert 'cgroup_dir / "cgroup.procs"' in script + assert "os.WUNTRACED" in script + assert "signal.SIGSTOP" in script + assert "signal.SIGCONT" in script + assert "os.waitpid(child, 0)" in script + assert "PR_SET_PDEATHSIG" in script + assert "signal.pthread_sigmask" in script + assert "ParentInterrupted" in script + assert "kill_and_reap(child)" in script + assert "Seccomp_filters" in script + assert '"/proc/$main_pid/attr/current"' in script + assert "/proc/1/ns/user" in script + assert "resource.prlimit" in script + assert "copy_process_limits" in script + assert "configure_stopped_child" in script assert 'preflight_service_before="$(service_fingerprint)"' in script assert 'assert_service_fingerprint "$preflight_service_before"' in script assert "backup_path" in script @@ -253,7 +317,7 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert 'REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"' in script assert 'sudo install -d -o root -g root -m 0755 "$runtime_parent"' in script assert 'sudo install -d -o root -g root -m 0755 "$runtime_bin"' in script - assert 'remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)"' in script + assert 'remote_tmp="$(sudo mktemp -d /run/teleo-gcp-leoclean-runtime.XXXXXX)"' in script assert 'sudo install -d -o root -g root -m 0755 "$payload_dir"' in script assert 'sudo install -d -o root -g root -m 0700 "$backup_dir"' in script assert 'sudo install -d -o teleo -g teleo -m 0700 "$work_dir"' in script @@ -288,7 +352,10 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert 'for executable_dir in \\\n /usr/local/sbin \\' in script assert 'sudo stat -Lc \'%U:%G\' "$executable_dir"' in script assert 'sudo -n -u teleo test ! -w "$executable_dir"' in script - assert 'for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3' in script + assert ( + 'for protected_file in /bin/bash /usr/bin/curl /usr/bin/env /usr/bin/gcloud ' + '/usr/bin/psql /usr/bin/python3 /usr/bin/setpriv' in script + ) assert '/usr/bin/curl -q --noproxy' in script assert 'test "$(sudo cat "$revision")" = "$SOURCE_REVISION"' in script assert 'assert_existing_root_directory "$existing_directory"' in script @@ -314,30 +381,39 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped(): assert 'sudo install -o root -g root -m 0755 "$work_dir/' not in script assert 'nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb"' in script assert 'findmnt -T "$SERVICE_PROFILE_BIN"' in script - assert 'findmnt -M "$SERVICE_HERMES_ROOT"' in script + assert "findmnt -M /home/teleo" in script assert 'findmnt -M "$SERVICE_PROFILE_ROOT/state"' in script assert 'findmnt -M "$SERVICE_PROFILE_ROOT/workspace"' in script - assert "cap_sys_admin" in script + assert "for capability_field in CapInh CapPrm CapEff CapBnd CapAmb" in script + assert "0000000000000000" in script assert '"current_tier": "T3_live_readonly"' in script assert '"status": "pass"' in script assert "UnsetEnvironment=PGPASSWORD" in dropin - assert "UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG" in dropin + assert ( + "UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CORE_PROJECT " + "CLOUDSDK_AUTH_ACCESS_TOKEN" in dropin + ) assert "UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS" in dropin assert "UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT" in dropin assert "UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT" in dropin assert "UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY" in dropin assert "UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE" in dropin + assert ( + "UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL " + "TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT" in dropin + ) assert ( "BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin" in dropin ) - assert "ReadOnlyPaths=/home/teleo/.hermes" in dropin + assert "ReadOnlyPaths=/home/teleo" in dropin assert ( "ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state " "/home/teleo/.hermes/profiles/leoclean/workspace" in dropin ) - assert "CapabilityBoundingSet=~CAP_SYS_ADMIN" in dropin + assert "CapabilityBoundingSet=" in dropin + assert "AmbientCapabilities=" in dropin assert "RuntimeDirectory=" not in dropin - assert "Environment=CLOUDSDK_" not in dropin + assert "Environment=CLOUDSDK_CONFIG=/usr/local/libexec/livingip/leoclean-kb/gcloud-config" in dropin assert "Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" in dropin assert "TELEO_KB_MODE=cloudsql" in dropin assert "TELEO_GCP_METADATA_ONLY=1" in dropin @@ -374,7 +450,7 @@ def test_gcp_runtime_embedded_remote_programs_are_bash_syntax_valid() -> None: assert completed.returncode == 0, f"{name}: {completed.stderr}" embedded_python = [part.split("\nPY", maxsplit=1)[0] for part in blocks["remote_helpers"].split("<<'PY'\n")[1:]] - assert len(embedded_python) == 2 + assert len(embedded_python) == 4 for index, body in enumerate(embedded_python): completed = subprocess.run( ["python3", "-c", "import sys; compile(sys.stdin.read(), '', 'exec')"], @@ -386,6 +462,162 @@ def test_gcp_runtime_embedded_remote_programs_are_bash_syntax_valid() -> None: assert completed.returncode == 0, f"embedded_python_{index}: {completed.stderr}" +@pytest.mark.parametrize( + ("command", "expected_returncode"), + [ + (["/bin/sh", "-c", "exit 0"], 0), + (["/bin/sh", "-c", "exit 7"], 7), + (["/bin/sh", "-c", "kill -TERM $$"], 128 + signal.SIGTERM), + ], +) +def test_service_context_runner_propagates_result_and_reaps_child( + command: list[str], + expected_returncode: int, +) -> None: + run_stopped_child = load_service_context_runner()["run_stopped_child"] + moved_children: list[int] = [] + + result = run_stopped_child( # type: ignore[operator] + command, + "/test.service", + move_child=moved_children.append, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + ) + + assert result == expected_returncode + assert len(moved_children) == 1 + with pytest.raises(ProcessLookupError): + os.kill(moved_children[0], 0) + + +def test_service_context_runner_reaps_stopped_child_when_move_fails() -> None: + run_stopped_child = load_service_context_runner()["run_stopped_child"] + stopped_children: list[int] = [] + + def fail_move(child: int) -> None: + stopped_children.append(child) + raise RuntimeError("synthetic move failure") + + with pytest.raises(RuntimeError, match="synthetic move failure"): + run_stopped_child( # type: ignore[operator] + ["/bin/sh", "-c", "exit 0"], + "/test.service", + move_child=fail_move, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + ) + + assert len(stopped_children) == 1 + with pytest.raises(ProcessLookupError): + os.kill(stopped_children[0], 0) + + +@pytest.mark.skipif(not hasattr(resource, "prlimit"), reason="requires Linux prlimit") +def test_service_context_runner_copies_mainpid_resource_limits(tmp_path: Path) -> None: + namespace = load_service_context_runner() + run_stopped_child = namespace["run_stopped_child"] + copy_process_limits = namespace["copy_process_limits"] + limit_readback = tmp_path / "limits.txt" + child_program = ( + "import resource,sys; from pathlib import Path; " + "limits=resource.getrlimit(resource.RLIMIT_NOFILE); " + "Path(sys.argv[1]).write_text(f'{limits[0]}:{limits[1]}', encoding='ascii')" + ) + + def lower_nofile_limit() -> None: + resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) + + source = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)"], + preexec_fn=lower_nofile_limit, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + result = run_stopped_child( # type: ignore[operator] + [sys.executable, "-c", child_program, str(limit_readback)], + "/test.service", + move_child=lambda _child: None, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + configure_stopped_child=lambda child: copy_process_limits(source.pid, child), # type: ignore[operator] + ) + finally: + source.terminate() + source.wait(timeout=5) + + assert result == 0 + assert limit_readback.read_text(encoding="ascii") == "64:64" + + +@pytest.mark.parametrize("signum", [signal.SIGTERM, signal.SIGHUP]) +def test_service_context_runner_interruption_kills_and_reaps_probe( + tmp_path: Path, + signum: signal.Signals, +) -> None: + run_stopped_child = load_service_context_runner()["run_stopped_child"] + child_pid_path = tmp_path / f"probe-{signum.name}.pid" + child_program = ( + "import os,sys,time; from pathlib import Path; " + "Path(sys.argv[1]).write_text(str(os.getpid()), encoding='ascii'); " + "time.sleep(30)" + ) + runner_pid = os.fork() + if runner_pid == 0: + result = run_stopped_child( # type: ignore[operator] + [sys.executable, "-c", child_program, str(child_pid_path)], + "/test.service", + move_child=lambda _child: None, + read_child_cgroup=lambda _pid: "/test.service", + prepare_child=lambda _parent: None, + ) + os._exit(result) + + runner_reaped = False + probe_pid: int | None = None + try: + deadline = time.monotonic() + 5 + probe_pid_text = "" + while time.monotonic() < deadline: + if child_pid_path.is_file(): + probe_pid_text = child_pid_path.read_text(encoding="ascii") + if probe_pid_text.isdecimal(): + break + waited, _status = os.waitpid(runner_pid, os.WNOHANG) + if waited == runner_pid: + runner_reaped = True + pytest.fail("service-context runner exited before probe start") + time.sleep(0.01) + assert probe_pid_text.isdecimal() + probe_pid = int(probe_pid_text) + + os.kill(runner_pid, signum) + waited, status = os.waitpid(runner_pid, 0) + runner_reaped = True + assert waited == runner_pid + assert os.WIFSIGNALED(status) + assert os.WTERMSIG(status) == signum + with pytest.raises(ProcessLookupError): + os.kill(probe_pid, 0) + finally: + if not runner_reaped: + try: + os.kill(runner_pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.waitpid(runner_pid, 0) + except ChildProcessError: + pass + if probe_pid is not None: + try: + os.kill(probe_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_gcp_runtime_rejects_preexisting_symlink_before_mutation(tmp_path: Path) -> None: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() function_start = script.index("assert_existing_root_directory() {") @@ -428,10 +660,43 @@ set -euo pipefail SERVICE=leoclean.service REMOTE_DROPIN_DIR=/etc/systemd/system/leoclean.service.d DROPIN_NAME=10-cloudsql-memory.conf +REMOTE_RUNTIME_BIN=/usr/local/libexec/livingip/leoclean-kb +SERVICE_PROFILE_ROOT=/home/teleo/.hermes/profiles/leoclean +SERVICE_PROFILE_BIN=$SERVICE_PROFILE_ROOT/bin +assert_exact_word_set() {{ + local actual="$1" expected="$2" label="$3" actual_sorted expected_sorted word + actual_sorted="$(for word in $actual; do printf '%s\\n' "${{word#-}}"; done | sort)" + expected_sorted="$(for word in $expected; do printf '%s\\n' "${{word#-}}"; done | sort)" + [ "$actual_sorted" = "$expected_sorted" ] || {{ + echo "Service effective $label differs from the reviewed exact allowlist." >&2 + return 1 + }} +}} systemctl() {{ - case "$*" in - *--property=DropInPaths*) printf '%s\\n' "${{FAKE_DROPINS:-}}" ;; - *--property=EnvironmentFiles*) printf '%s\\n' "${{FAKE_ENVIRONMENT_FILES:-}}" ;; + local argument property= + for argument in "$@"; do + case "$argument" in --property=*) property=${{argument#--property=}} ;; esac + done + case "$property" in + DropInPaths) printf '%s\\n' "${{FAKE_DROPINS:-}}" ;; + EnvironmentFiles) printf '%s\\n' "${{FAKE_ENVIRONMENT_FILES:-}}" ;; + LoadCredential|LoadCredentialEncrypted|SetCredential|SetCredentialEncrypted|ImportCredential|ExecCondition|ExecStartPre|ExecStartPost|ExecReload|ExecStop|ExecStopPost) + local variable="FAKE_$property" + printf '%s\\n' "${{!variable:-}}" + ;; + CapabilityBoundingSet|AmbientCapabilities|BindPaths|TemporaryFileSystem) printf '\\n' ;; + NoNewPrivileges) printf 'yes\\n' ;; + PrivateNetwork) printf '%s\\n' "${{FAKE_PrivateNetwork:-no}}" ;; + PrivateUsers) printf '%s\\n' "${{FAKE_PrivateUsers:-no}}" ;; + NetworkNamespacePath|IPAddressDeny|IPAddressAllow|RestrictAddressFamilies|RestrictNetworkInterfaces|SocketBindAllow|SocketBindDeny|SystemCallFilter|AppArmorProfile|SELinuxContext) + local variable="FAKE_$property" + printf '%s\\n' "${{!variable:-}}" + ;; + Group|SupplementaryGroups) printf 'teleo\\n' ;; + ReadOnlyPaths) printf '%s\\n' "${{FAKE_ReadOnlyPaths:-/home/teleo}}" ;; + ReadWritePaths) printf '%s\\n' "$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" ;; + BindReadOnlyPaths) printf '%s\\n' "$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" ;; + InaccessiblePaths) printf '%s\\n' '/home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql' ;; *) return 90 ;; esac }} @@ -445,6 +710,19 @@ assert_unit_configuration "FAKE_ENVIRONMENT_FILES": "", }, {"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": "/etc/leoclean-runtime.env"}, + {"FAKE_DROPINS": reviewed, "FAKE_ExecStartPost": "/usr/local/bin/conflicting-hook"}, + {"FAKE_DROPINS": reviewed, "FAKE_ReadOnlyPaths": "/home/teleo /tmp"}, + {"FAKE_DROPINS": reviewed, "FAKE_PrivateNetwork": "yes"}, + {"FAKE_DROPINS": reviewed, "FAKE_PrivateUsers": "yes"}, + {"FAKE_DROPINS": reviewed, "FAKE_NetworkNamespacePath": "/run/netns/restricted"}, + {"FAKE_DROPINS": reviewed, "FAKE_IPAddressDeny": "any"}, + {"FAKE_DROPINS": reviewed, "FAKE_IPAddressAllow": "10.61.0.4/32"}, + {"FAKE_DROPINS": reviewed, "FAKE_RestrictAddressFamilies": "AF_UNIX"}, + {"FAKE_DROPINS": reviewed, "FAKE_RestrictNetworkInterfaces": "~ens4"}, + {"FAKE_DROPINS": reviewed, "FAKE_SocketBindDeny": "any"}, + {"FAKE_DROPINS": reviewed, "FAKE_SystemCallFilter": "@system-service ~@network-io"}, + {"FAKE_DROPINS": reviewed, "FAKE_AppArmorProfile": "restricted-profile"}, + {"FAKE_DROPINS": reviewed, "FAKE_SELinuxContext": "system_u:system_r:restricted_t:s0"}, ) for environment in cases: completed = subprocess.run( @@ -462,12 +740,53 @@ assert_unit_configuration input=harness, capture_output=True, check=False, - env={"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": ""}, + env={"FAKE_DROPINS": reviewed}, text=True, ) assert allowed.returncode == 0, allowed.stderr +def test_gcp_runtime_effective_execstart_override_is_rejected_before_process_probe() -> None: + script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() + function_start = script.index("assert_service_running() {") + function_end = script.index("\n\nassert_unit_configuration() {", function_start) + service_function = script[function_start:function_end] + harness = f""" +set -euo pipefail +SERVICE=leoclean.service +EXPECTED_HERMES_EXECUTABLE=/home/teleo/.hermes/hermes-agent/hermes.py +EXPECTED_HERMES_PYTHON=/home/teleo/.hermes/hermes-agent/venv/bin/python +systemctl() {{ + local argument property= + for argument in "$@"; do + case "$argument" in --property=*) property=${{argument#--property=}} ;; esac + done + case "$property" in + ActiveState) printf 'active\\n' ;; + SubState) printf 'running\\n' ;; + MainPID) printf '1234\\n' ;; + User) printf 'teleo\\n' ;; + ExecStart) printf '{{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /tmp/conflicting.py ; }}\\n' ;; + *) return 90 ;; + esac +}} +ps() {{ printf 'teleo\\n'; }} +{service_function} +assert_service_running false +""" + + completed = subprocess.run( + ["bash", "-s"], + input=harness, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode != 0 + assert "effective ExecStart is not the reviewed" in completed.stderr + + def test_gcp_runtime_absence_rollback_removes_every_created_directory(tmp_path: Path) -> None: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() function_start = script.index("backup_directory_state() {")