#!/usr/bin/env bash # Deploy the source-controlled Cloud SQL bridge and non-secret service settings # to the GCP parallel leoclean runtime. This does not create database users, # read secret values, send Telegram messages, or promote Cloud SQL. set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" PROJECT="${PROJECT:-teleo-501523}" ZONE="${ZONE:-europe-west6-a}" INSTANCE="${INSTANCE:-teleo-prod-1}" SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}" MERGED_REF="${MERGED_REF:-origin/main}" REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}" SERVICE_PROFILE_BIN="${SERVICE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}" SERVICE_PROFILE_ROOT="${SERVICE_PROFILE_ROOT:-/home/teleo/.hermes/profiles/leoclean}" SERVICE_HERMES_ROOT="${SERVICE_HERMES_ROOT:-/home/teleo/.hermes}" REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}" EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}" EXPECTED_HERMES_EXECUTABLE="${EXPECTED_HERMES_EXECUTABLE:-/home/teleo/.hermes/hermes-agent/venv/bin/hermes}" EXPECTED_HERMES_PYTHON="${EXPECTED_HERMES_PYTHON:-/home/teleo/.hermes/hermes-agent/venv/bin/python3}" WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb-gcp" TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py" PERMISSION_VERIFIER_REL="ops/verify_gcp_leoclean_runtime_permissions.py" SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py" SERVER_CA_REL="ops/gcp-teleo-pgvector-standby-server-ca.pem" DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf" DROPIN_NAME="10-cloudsql-memory.conf" DRY_RUN=false RESTART=false VERIFY_ONLY=false PREFLIGHT_ONLY=false usage() { cat <<'USAGE' Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--preflight-only] [--restart] [--verify-only] The database role and scoped Secret Manager secret must exist before --restart. This script deploys only the bridge files and non-secret systemd drop-in. USAGE } for arg in "$@"; do case "$arg" in --dry-run) DRY_RUN=true ;; --preflight-only) PREFLIGHT_ONLY=true ;; --restart) RESTART=true ;; --verify-only) VERIFY_ONLY=true ;; --help|-h) usage exit 0 ;; *) echo "Unknown arg: $arg" >&2 usage >&2 exit 64 ;; esac done if $PREFLIGHT_ONLY && $RESTART; then echo "--preflight-only and --restart are mutually exclusive." >&2 exit 64 fi if $VERIFY_ONLY && { $PREFLIGHT_ONLY || $RESTART; }; then echo "--verify-only cannot be combined with --preflight-only or --restart." >&2 exit 64 fi for path in \ "$WRAPPER_REL" \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ "$SERVER_CA_REL" \ "$DROPIN_REL"; do if [ ! -f "$REPO_ROOT/$path" ]; then echo "Required source file is missing: $path" >&2 exit 66 fi if [ -L "$REPO_ROOT/$path" ]; then echo "Refusing symlinked runtime source file: $path" >&2 exit 65 fi done if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2 exit 65 fi if ! grep -Fxq 'Environment=TELEO_GCP_METADATA_ONLY=1' "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in must enforce metadata-only GCP credentials." >&2 exit 65 fi if ! grep -Fxq "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in does not bind the root-controlled runtime into the service profile." >&2 exit 65 fi if ! grep -Fxq 'ReadOnlyPaths=/home/teleo' "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in does not make the complete service home read-only." >&2 exit 65 fi if ! grep -Fxq "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in does not limit service-profile writes to state and workspace." >&2 exit 65 fi if ! grep -Fxq 'CapabilityBoundingSet=' "$REPO_ROOT/$DROPIN_REL" || ! grep -Fxq 'AmbientCapabilities=' "$REPO_ROOT/$DROPIN_REL" || ! grep -Fxq 'NoNewPrivileges=yes' "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in must remove all capabilities and enforce no-new-privileges." >&2 exit 65 fi if ! grep -Fxq 'Group=teleo' "$REPO_ROOT/$DROPIN_REL" || ! grep -Fxq 'SupplementaryGroups=teleo' "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in must bind the exact service group set." >&2 exit 65 fi if ! grep -Fxq 'UnsetEnvironment=OPENSSL_CONF OPENSSL_MODULES OPENSSL_ENGINES GCONV_PATH CREDENTIALS_DIRECTORY' "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in must remove startup-loader and systemd-credential injection fields." >&2 exit 65 fi if ! grep -Fxq 'InaccessiblePaths=-/home/teleo/.config/gcloud -/home/teleo/.pgpass -/home/teleo/.pg_service.conf -/home/teleo/.postgresql' "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in must mask default GCP and PostgreSQL credential paths." >&2 exit 65 fi if ! grep -Fxq \ 'UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT' \ "$REPO_ROOT/$DROPIN_REL"; then echo "Runtime drop-in must unset optional Cloud SQL and memory controls." >&2 exit 65 fi SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)" if ! git -C "$REPO_ROOT" diff --quiet HEAD -- \ "$WRAPPER_REL" \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ "$SERVER_CA_REL" \ "$DROPIN_REL"; then echo "Refusing to deploy runtime files that do not match HEAD." >&2 exit 65 fi if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- \ "$WRAPPER_REL" \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ "$SERVER_CA_REL" \ "$DROPIN_REL")" ]; then echo "Refusing to deploy untracked runtime files." >&2 exit 65 fi if ! git -C "$REPO_ROOT" rev-parse --verify "$MERGED_REF^{commit}" >/dev/null 2>&1; then echo "Merged reference is unavailable: $MERGED_REF" >&2 exit 69 fi if ! git -C "$REPO_ROOT" merge-base --is-ancestor "$SOURCE_REVISION" "$MERGED_REF"; then echo "Refusing to deploy unmerged revision $SOURCE_REVISION (not contained in $MERGED_REF)." >&2 exit 65 fi WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')" TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')" PERMISSION_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" SERVICE_ENV_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" SERVER_CA_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVER_CA_REL" | awk '{print $1}')" DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')" GCP_SSH=( gcloud compute ssh "$INSTANCE" --project="$PROJECT" --zone="$ZONE" --tunnel-through-iap ) remote_helpers=$(cat <<'REMOTE' assert_exact_word_set() { local actual="$1" expected="$2" label="$3" actual_sorted expected_sorted word actual_sorted="$(for word in $actual; do printf '%s\n' "${word#-}"; done | sort)" expected_sorted="$(for word in $expected; do printf '%s\n' "${word#-}"; done | sort)" if [ "$actual_sorted" != "$expected_sorted" ]; then echo "Service effective $label differs from the reviewed exact allowlist." >&2 return 1 fi } assert_service_running() { local require_reviewed="${1:-true}" local active_state sub_state main_pid unit_user process_user teleo_uid teleo_gid local uid_fields gid_fields groups_field exec_start process_executable local seccomp_mode seccomp_filters lsm_context local host_user_namespace service_user_namespace active_state="$(systemctl show "$SERVICE" --property=ActiveState --value)" sub_state="$(systemctl show "$SERVICE" --property=SubState --value)" if [ "$active_state" != "active" ] || [ "$sub_state" != "running" ]; then echo "Service is not active/running: ActiveState=$active_state SubState=$sub_state" >&2 return 1 fi main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" unit_user="$(systemctl show "$SERVICE" --property=User --value)" case "$main_pid" in ''|0|*[!0-9]*) echo "Service has no live MainPID: $main_pid" >&2 return 1 ;; esac if [ "$unit_user" != "teleo" ]; then echo "Service unit user is not teleo: $unit_user" >&2 return 1 fi process_user="$(ps -o user= -p "$main_pid" | awk '{print $1}')" if [ "$process_user" != "teleo" ]; then echo "Service process user is not teleo: $process_user" >&2 return 1 fi if [ "$require_reviewed" = true ]; then teleo_uid="$(id -u teleo)" teleo_gid="$(id -g teleo)" uid_fields="$(awk '/^Uid:/{print $2 ":" $3 ":" $4 ":" $5}' "/proc/$main_pid/status")" gid_fields="$(awk '/^Gid:/{print $2 ":" $3 ":" $4 ":" $5}' "/proc/$main_pid/status")" groups_field="$(awk '/^Groups:/{for (i=2;i<=NF;i++) printf "%s%s", (i==2 ? "" : ":"), $i}' "/proc/$main_pid/status")" if [ "$uid_fields" != "$teleo_uid:$teleo_uid:$teleo_uid:$teleo_uid" ] || [ "$gid_fields" != "$teleo_gid:$teleo_gid:$teleo_gid:$teleo_gid" ] || [ "$groups_field" != "$teleo_gid" ]; then echo "Service process does not have the exact reviewed uid/gid/group set." >&2 return 1 fi fi exec_start="$(systemctl show "$SERVICE" --property=ExecStart --value)" case "$exec_start" in *"path=$EXPECTED_HERMES_EXECUTABLE ; argv[]=$EXPECTED_HERMES_EXECUTABLE -p leoclean gateway run ;"*) ;; *) echo "Service effective ExecStart is not the reviewed leoclean Hermes gateway." >&2 return 1 ;; esac sudo /usr/bin/python3 -I - "$main_pid" "$EXPECTED_HERMES_PYTHON" "$EXPECTED_HERMES_EXECUTABLE" <<'PY' import os import sys from pathlib import Path pid, expected_python, expected_hermes = sys.argv[1:] raw = Path(f"/proc/{pid}/cmdline").read_bytes() if not raw.endswith(b"\0") or b"\0\0" in raw: raise SystemExit("service cmdline is malformed") try: argv = [part.decode("utf-8", "strict") for part in raw[:-1].split(b"\0")] except UnicodeDecodeError: raise SystemExit("service cmdline is not valid UTF-8") from None if argv != [expected_python, expected_hermes, "-p", "leoclean", "gateway", "run"]: raise SystemExit("service cmdline is not the reviewed leoclean Hermes gateway") if os.path.realpath(f"/proc/{pid}/exe") != os.path.realpath(expected_python): raise SystemExit("service executable is not the reviewed Hermes interpreter") PY if [ "$require_reviewed" = true ]; then for capability_field in CapInh CapPrm CapEff CapBnd CapAmb; do if [ "$(awk -v key="$capability_field:" '$1 == key {print $2}' "/proc/$main_pid/status")" != 0000000000000000 ]; then echo "Service process retains capabilities in $capability_field." >&2 return 1 fi done if [ "$(awk '/^NoNewPrivs:/{print $2}' "/proc/$main_pid/status")" != 1 ]; then echo "Service process does not enforce no-new-privileges." >&2 return 1 fi seccomp_mode="$(awk '/^Seccomp:/{print $2}' "/proc/$main_pid/status")" seccomp_filters="$(awk '/^Seccomp_filters:/{print $2}' "/proc/$main_pid/status")" lsm_context="$(sudo cat "/proc/$main_pid/attr/current")" host_user_namespace="$(sudo stat -Lc '%d:%i' /proc/1/ns/user)" service_user_namespace="$(sudo stat -Lc '%d:%i' "/proc/$main_pid/ns/user")" if [ "$seccomp_mode" != 0 ] || { [ -n "$seccomp_filters" ] && [ "$seccomp_filters" != 0 ]; } || [ "$lsm_context" != unconfined ] || [ "$service_user_namespace" != "$host_user_namespace" ]; then echo "Service process has an unreviewed live seccomp, LSM, or user-namespace restriction." >&2 return 1 fi fi } assert_unit_configuration() { local require_reviewed="${1:-true}" local dropin_path dropin_name dropin_paths environment_files reviewed_dropin_seen=false local property value capability_set ambient_capabilities no_new_privileges unit_group supplementary_groups local inaccessible_paths credential_path local read_only_paths read_write_paths bind_paths bind_read_only_paths temporary_file_systems local private_network private_users network_property dropin_paths="$(systemctl show "$SERVICE" --property=DropInPaths --value)" environment_files="$(systemctl show "$SERVICE" --property=EnvironmentFiles --value)" if [ -n "$environment_files" ]; then echo "Service has an effective EnvironmentFile, which is not allowed for the scoped runtime." >&2 return 1 fi for property in LoadCredential LoadCredentialEncrypted SetCredential SetCredentialEncrypted ImportCredential ExecCondition ExecStartPre ExecStartPost ExecReload ExecStop ExecStopPost; do value="$(systemctl show "$SERVICE" --property="$property" --value)" if [ -n "$value" ]; then echo "Service has an effective $property credential source." >&2 return 1 fi done for dropin_path in $dropin_paths; do if [ "$dropin_path" = "$REMOTE_DROPIN_DIR/$DROPIN_NAME" ]; then reviewed_dropin_seen=true continue fi dropin_name="${dropin_path##*/}" if [[ "$dropin_name" > "$DROPIN_NAME" ]]; then echo "Service has a later drop-in than the reviewed Cloud SQL configuration: $dropin_name" >&2 return 1 fi done if [ "$require_reviewed" = true ] && [ "$reviewed_dropin_seen" != true ]; then echo "Service does not load the reviewed Cloud SQL drop-in." >&2 return 1 fi if [ "$require_reviewed" = true ]; then capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)" ambient_capabilities="$(systemctl show "$SERVICE" --property=AmbientCapabilities --value)" no_new_privileges="$(systemctl show "$SERVICE" --property=NoNewPrivileges --value)" unit_group="$(systemctl show "$SERVICE" --property=Group --value)" supplementary_groups="$(systemctl show "$SERVICE" --property=SupplementaryGroups --value)" inaccessible_paths="$(systemctl show "$SERVICE" --property=InaccessiblePaths --value)" read_only_paths="$(systemctl show "$SERVICE" --property=ReadOnlyPaths --value)" read_write_paths="$(systemctl show "$SERVICE" --property=ReadWritePaths --value)" bind_paths="$(systemctl show "$SERVICE" --property=BindPaths --value)" bind_read_only_paths="$(systemctl show "$SERVICE" --property=BindReadOnlyPaths --value)" temporary_file_systems="$(systemctl show "$SERVICE" --property=TemporaryFileSystem --value)" if [ -n "$capability_set" ] || [ -n "$ambient_capabilities" ] || [ "$no_new_privileges" != yes ] || [ "$unit_group" != teleo ] || [ "$supplementary_groups" != teleo ]; then echo "Service effective capability, privilege, or group contract differs from the reviewed drop-in." >&2 return 1 fi assert_exact_word_set "$read_only_paths" "/home/teleo" ReadOnlyPaths assert_exact_word_set "$read_write_paths" "$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" ReadWritePaths assert_exact_word_set "$bind_paths" "" BindPaths assert_exact_word_set "$bind_read_only_paths" "$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" BindReadOnlyPaths assert_exact_word_set "$temporary_file_systems" "" TemporaryFileSystem assert_exact_word_set "$supplementary_groups" "teleo" SupplementaryGroups assert_exact_word_set "$inaccessible_paths" "/home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql" InaccessiblePaths private_network="$(systemctl show "$SERVICE" --property=PrivateNetwork --value)" private_users="$(systemctl show "$SERVICE" --property=PrivateUsers --value)" if [ "$private_network" != no ] || [ "$private_users" != no ]; then echo "Service effective private network or user namespace differs from the reviewed host contract." >&2 return 1 fi for network_property in \ NetworkNamespacePath \ IPAddressDeny \ IPAddressAllow \ RestrictAddressFamilies \ RestrictNetworkInterfaces \ SocketBindAllow \ SocketBindDeny \ SystemCallFilter \ AppArmorProfile \ SELinuxContext; do value="$(systemctl show "$SERVICE" --property="$network_property" --value)" if [ -n "$value" ]; then echo "Service effective $network_property differs from the reviewed unrestricted transport contract." >&2 return 1 fi done for credential_path in /home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql; do case " $inaccessible_paths " in *" $credential_path "*) ;; *) echo "Service does not mask a default credential path: $credential_path" >&2 return 1 ;; esac done fi } assert_runtime_environment() { local verifier_path="$1" require_reviewed="${2:-true}" main_pid main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" if [ "$require_reviewed" = true ]; then sudo /usr/bin/python3 -I "$verifier_path" --pid "$main_pid" else sudo /usr/bin/python3 -I "$verifier_path" --pid "$main_pid" --allow-missing-runtime-security-fields fi assert_unit_configuration "$require_reviewed" if [ "$require_reviewed" = true ]; then sudo nsenter -t "$main_pid" -m -- test ! -e "/run/credentials/$SERVICE" for credential_path in /home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql; do sudo nsenter -t "$main_pid" -m -- \ /usr/bin/setpriv --reuid=teleo --regid=teleo --clear-groups --no-new-privs \ /usr/bin/test ! -r "$credential_path" done fi } assert_attached_service_account() { local metadata_email metadata_email="$( sudo -n -u teleo env -i \ HOME=/home/teleo \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ /usr/bin/curl -q --noproxy '*' --proto '=http' --max-time 10 --fail --silent --show-error \ --header 'Metadata-Flavor: Google' \ 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email' )" if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then echo "Attached VM service account mismatch: $metadata_email" >&2 return 1 fi echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email" } run_in_service_cgroup() { local main_pid="$1" control_group shift control_group="$(systemctl show "$SERVICE" --property=ControlGroup --value)" if [ -z "$control_group" ]; then echo "Service has no effective cgroup." >&2 return 1 fi sudo /usr/bin/python3 -I - "$main_pid" "$control_group" "$@" <<'PY' import ctypes import os import resource import signal import stat import sys from pathlib import Path, PurePosixPath HANDLED_SIGNALS = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP) PR_SET_PDEATHSIG = 1 class ParentInterrupted(BaseException): def __init__(self, signum: int) -> None: self.signum = signum def fail() -> None: raise SystemExit("service-context runner failed closed") def process_cgroup(pid: str) -> str: try: lines = Path(f"/proc/{pid}/cgroup").read_text(encoding="ascii").splitlines() except (OSError, UnicodeError): fail() matches = [] for line in lines: parts = line.split(":", 2) if len(parts) == 3 and parts[0] == "0" and parts[1] == "": matches.append(parts[2]) if len(matches) != 1: fail() value = matches[0] path = PurePosixPath(value) if not value.startswith("/") or ".." in path.parts or str(path) != value: fail() return value def set_parent_death_signal(expected_parent: int) -> None: libc = ctypes.CDLL(None, use_errno=True) prctl = libc.prctl prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] prctl.restype = ctypes.c_int if prctl(PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) != 0: os._exit(125) if os.getppid() != expected_parent: os._exit(125) def kill_and_reap(child: int) -> None: try: os.kill(child, signal.SIGKILL) except ProcessLookupError: pass while True: try: os.waitpid(child, 0) return except InterruptedError: continue except ChildProcessError: return def supported_resource_ids() -> tuple[int, ...]: identifiers = set() for name in dir(resource): if not name.startswith("RLIMIT_") or name == "RLIMIT_NLIMITS": continue identifier = getattr(resource, name) if not isinstance(identifier, int) or identifier < 0: continue try: resource.prlimit(os.getpid(), identifier) except (OSError, ValueError): continue identifiers.add(identifier) return tuple(sorted(identifiers)) def copy_process_limits(source_pid: int, child: int) -> None: snapshot = { identifier: resource.prlimit(source_pid, identifier) for identifier in supported_resource_ids() } for identifier, limits in snapshot.items(): resource.prlimit(child, identifier, limits) if resource.prlimit(child, identifier) != limits: fail() if any(resource.prlimit(source_pid, identifier) != limits for identifier, limits in snapshot.items()): fail() def no_op_stopped_child(_child: int) -> None: return None def run_stopped_child( command: list[str], expected_cgroup: str, *, move_child, read_child_cgroup, prepare_child=set_parent_death_signal, configure_stopped_child=no_op_stopped_child, ) -> int: if not command or not command[0].startswith("/"): fail() original_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) parent_pid = os.getpid() try: child = os.fork() except BaseException: signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) raise if child == 0: try: for signum in HANDLED_SIGNALS: signal.signal(signum, signal.SIG_DFL) prepare_child(parent_pid) signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) if os.getppid() != parent_pid: os._exit(125) os.kill(os.getpid(), signal.SIGSTOP) os.execv(command[0], command) except BaseException: os._exit(126) previous_handlers = {} reaped = False interrupted = None status = None def interrupt(signum, _frame) -> None: signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) raise ParentInterrupted(signum) try: for signum in HANDLED_SIGNALS: previous_handlers[signum] = signal.signal(signum, interrupt) signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) waited, status = os.waitpid(child, os.WUNTRACED) if waited != child or not os.WIFSTOPPED(status) or os.WSTOPSIG(status) != signal.SIGSTOP: fail() move_child(child) if read_child_cgroup(str(child)) != expected_cgroup: fail() configure_stopped_child(child) os.kill(child, signal.SIGCONT) waited, status = os.waitpid(child, 0) if waited != child: fail() reaped = True except ParentInterrupted as exc: interrupted = exc.signum finally: signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) if not reaped: kill_and_reap(child) for signum, previous in previous_handlers.items(): signal.signal(signum, previous) signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) if interrupted is not None: signal.signal(interrupted, signal.SIG_DFL) os.kill(os.getpid(), interrupted) fail() if status is None: fail() if os.WIFEXITED(status): return os.WEXITSTATUS(status) if os.WIFSIGNALED(status): return 128 + os.WTERMSIG(status) fail() def main() -> int: if len(sys.argv) < 4: fail() main_pid, expected_cgroup, *command = sys.argv[1:] if not main_pid.isascii() or not main_pid.isdecimal() or int(main_pid) <= 0: fail() if process_cgroup(main_pid) != expected_cgroup: fail() cgroup_root = Path("/sys/fs/cgroup").resolve(strict=True) cgroup_dir = (cgroup_root / expected_cgroup.lstrip("/")).resolve(strict=True) try: cgroup_dir.relative_to(cgroup_root) except ValueError: fail() cgroup_procs = cgroup_dir / "cgroup.procs" try: if not stat.S_ISREG(cgroup_procs.stat().st_mode): fail() except OSError: fail() def move_child(child: int) -> None: with cgroup_procs.open("w", encoding="ascii") as handle: handle.write(f"{child}\n") return run_stopped_child( command, expected_cgroup, move_child=move_child, read_child_cgroup=process_cgroup, configure_stopped_child=lambda child: copy_process_limits(int(main_pid), child), ) if __name__ == "__main__": raise SystemExit(main()) PY } assert_administrator_secret_denied() { local main_pid probe_program main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" probe_program="$(cat <<'PY' import json import os import subprocess import sys import urllib.error import urllib.parse import urllib.request project, expected_email = sys.argv[1:] metadata_headers = {"Metadata-Flavor": "Google"} opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) def metadata(path: str) -> bytes: request = urllib.request.Request( "http://169.254.169.254/computeMetadata/v1/" + path, headers=metadata_headers, ) with opener.open(request, timeout=10) as response: return response.read(65536) try: email = metadata("instance/service-accounts/default/email").decode("utf-8").strip() if email != expected_email: raise RuntimeError token_payload = json.loads(metadata("instance/service-accounts/default/token")) token = token_payload.get("access_token") if token_payload.get("token_type") != "Bearer" or not isinstance(token, str) or not token: raise RuntimeError secret = urllib.parse.quote("gcp-teleo-pgvector-standby-postgres-password", safe="") url = f"https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{secret}/versions/latest:access" request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) try: response = opener.open(request, timeout=10) except urllib.error.HTTPError as error: body = error.read(65536) parsed = json.loads(body) detail = parsed.get("error", {}) message = str(detail.get("message", "")).lower() if error.code != 403 or detail.get("status") != "PERMISSION_DENIED": raise RuntimeError from None if "secretmanager.versions.access" not in message: raise RuntimeError else: response.close() raise RuntimeError gcloud_command = [ "/usr/bin/gcloud", "secrets", "versions", "access", "latest", "--secret=gcp-teleo-pgvector-standby-postgres-password", f"--project={project}", "--quiet", ] default_discovery_environment = os.environ.copy() for key in ( "CLOUDSDK_CONFIG", "GOOGLE_APPLICATION_CREDENTIALS", "PGPASSFILE", "PGSERVICE", "XDG_CONFIG_HOME", ): default_discovery_environment.pop(key, None) default_discovery_environment["HOME"] = "/home/teleo" for environment in (os.environ.copy(), default_discovery_environment): denied = subprocess.run( gcloud_command, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, env=environment, timeout=30, check=False, ) denial_error = denied.stderr[:65536] if denied.returncode == 0: raise RuntimeError if "PERMISSION_DENIED" not in denial_error or "secretmanager.versions.access" not in denial_error: raise RuntimeError except Exception: raise SystemExit("administrator secret was not the exact expected metadata-identity IAM denial") from None print(json.dumps({ "administrator_secret_access": "iam_permission_denied", "ambient_gcloud_administrator_secret_access": "iam_permission_denied", "credential_context": "systemd_main_pid_environment", "direct_metadata_administrator_secret_access": "iam_permission_denied", "default_discovery_administrator_secret_access": "iam_permission_denied", "identity_source": "gce_metadata_ambient_gcloud_and_default_discovery", "stdout_discarded": True, }, sort_keys=True)) PY )" run_in_service_cgroup "$main_pid" \ /usr/bin/nsenter -t "$main_pid" -m -n --env -- \ /usr/bin/setpriv \ --reuid=teleo \ --regid=teleo \ --clear-groups \ --no-new-privs \ /usr/bin/python3 -I -c "$probe_program" "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" } run_scoped_status() { local phase="$1" tool_path="$2" server_ca_path="${3:-$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem}" local cloudsdk_config="${4:-$REMOTE_RUNTIME_BIN/gcloud-config}" assert_attached_service_account echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical" sudo -n -u teleo env -i \ HOME=/home/teleo \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ TELEO_GCP_PROJECT=teleo-501523 \ TELEO_GCP_METADATA_ONLY=1 \ TELEO_CLOUDSQL_HOST=10.61.0.3 \ TELEO_CLOUDSQL_PORT=5432 \ TELEO_CLOUDSQL_DB=teleo_canonical \ TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical \ TELEO_CLOUDSQL_USER=leoclean_kb_runtime \ TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \ TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \ CLOUDSDK_CONFIG="$cloudsdk_config" \ PGSSLMODE=verify-ca \ PGSSLROOTCERT="$server_ca_path" \ TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ /usr/bin/python3 -I "$tool_path" status --format json --redacted } run_service_wrapper_status() { local phase="$1" main_pid assert_attached_service_account main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" echo "SCOPED_WRAPPER_STATUS phase=$phase user=teleo database=teleo_canonical" # --env imports the target MainPID's effective /proc environment. setpriv # drops back to the service's Unix identity without sudo resetting that env. run_in_service_cgroup "$main_pid" \ /usr/bin/nsenter -t "$main_pid" -m -n --env -- \ /usr/bin/setpriv \ --reuid=teleo \ --regid=teleo \ --clear-groups \ --no-new-privs \ "$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted } service_fingerprint() { printf '%s|%s|%s|%s' \ "$(systemctl show "$SERVICE" --property=ActiveState --value)" \ "$(systemctl show "$SERVICE" --property=SubState --value)" \ "$(systemctl show "$SERVICE" --property=MainPID --value)" \ "$(systemctl show "$SERVICE" --property=NRestarts --value)" } assert_service_fingerprint() { local expected="$1" actual actual="$(service_fingerprint)" if [ "$actual" != "$expected" ]; then echo "Service state changed during the read-only preflight." >&2 return 1 fi echo "PREFLIGHT_SERVICE_UNCHANGED fingerprint=$actual" } run_permission_verifier() { local phase="$1" verifier_path="$2" receipt_path="$3" local server_ca_path="${4:-$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem}" receipt_dir run_id receipt_state local main_pid local cloudsdk_config="${5:-$REMOTE_RUNTIME_BIN/gcloud-config}" run_id="$(date -u +%Y%m%d%H%M%S)-$phase" receipt_dir="$(dirname "$receipt_path")" sudo -n -u teleo mkdir -p -- "$receipt_dir" sudo -n -u teleo chmod 0700 "$receipt_dir" echo "NEGATIVE_PERMISSION_RECEIPT phase=$phase run_id=$run_id" if [ "$phase" = pre ]; then sudo -n -u teleo env -i \ HOME=/home/teleo \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ /usr/bin/python3 -I "$verifier_path" --run-id "$run_id" --output "$receipt_path" else main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" run_in_service_cgroup "$main_pid" \ /usr/bin/nsenter -t "$main_pid" -m -n -- \ /usr/bin/setpriv \ --reuid=teleo \ --regid=teleo \ --clear-groups \ --no-new-privs \ /usr/bin/env -i \ HOME=/home/teleo \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \ TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \ /usr/bin/python3 -I "$verifier_path" --run-id "$run_id" --output "$receipt_path" fi receipt_state="$(sudo stat -c '%U:%G:%a' "$receipt_path")" if [ "$receipt_state" != "teleo:teleo:600" ]; then echo "Permission receipt has unexpected ownership or mode: $receipt_state" >&2 return 1 fi sudo /usr/bin/python3 -I - "$receipt_path" <<'PY' import json import sys from pathlib import Path receipt = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) expected = { "artifact": "gcp_leoclean_runtime_permissions", "current_tier": "T3_live_readonly", "mode": "live_private_gcp_staging", "required_tier": "T3_live_readonly", "status": "pass", } mismatched = sorted(key for key, value in expected.items() if receipt.get(key) != value) if mismatched: raise SystemExit("permission receipt contract mismatch: " + ",".join(mismatched)) print(json.dumps({"permission_receipt": "validated", "validated_fields": sorted(expected)}, sort_keys=True)) PY } assert_trusted_executable() { local requested="$1" resolved current owner sudo test -x "$requested" owner="$(sudo stat -c '%U' "$requested")" if [ "$owner" != root ]; then echo "Runtime executable path is not root-owned: $requested" >&2 return 1 fi resolved="$(sudo readlink -f -- "$requested")" sudo test -f "$resolved" sudo test -x "$resolved" current="$(dirname "$requested")" while :; do test "$(sudo stat -Lc '%U' "$current")" = root sudo -n -u teleo test ! -w "$current" [ "$current" = / ] && break current="$(dirname "$current")" done current="$resolved" while :; do test "$(sudo stat -Lc '%U' "$current")" = root sudo -n -u teleo test ! -w "$current" [ "$current" = / ] && break current="$(dirname "$current")" done } assert_installed_files() { local wrapper tool permission_verifier service_env_verifier server_ca gcloud_config dropin revision local executable_dir protected_dir protected_file actual_entries expected_entries wrapper="$REMOTE_RUNTIME_BIN/teleo-kb" tool="$REMOTE_RUNTIME_BIN/cloudsql_memory_tool.py" permission_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" service_env_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" server_ca="$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem" gcloud_config="$REMOTE_RUNTIME_BIN/gcloud-config" dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME" revision="$REMOTE_RUNTIME_BIN/.livingip-runtime-revision" test "$(sudo sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA" test "$(sudo sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA" test "$(sudo sha256sum "$permission_verifier" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" test "$(sudo sha256sum "$service_env_verifier" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" test "$(sudo sha256sum "$server_ca" | awk '{print $1}')" = "$SERVER_CA_SHA" test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA" test "$(sudo stat -c '%U:%G:%a' "$REMOTE_RUNTIME_BIN")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$wrapper")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$tool")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$permission_verifier")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$service_env_verifier")" = "root:root:755" test "$(sudo stat -c '%U:%G:%a' "$server_ca")" = "root:root:644" test "$(sudo stat -c '%U:%G:%a' "$gcloud_config")" = "root:root:555" test "$(sudo stat -c '%U:%G:%a' "$revision")" = "root:root:644" test "$(sudo stat -c '%U:%G:%a' "$dropin")" = "root:root:644" test "$(sudo cat "$revision")" = "$SOURCE_REVISION" sudo test -z "$(sudo find "$gcloud_config" -mindepth 1 -print -quit)" expected_entries="$(printf '%s\n' .livingip-runtime-revision cloudsql-server-ca.pem cloudsql_memory_tool.py gcloud-config teleo-kb verify_gcp_leoclean_runtime_permissions.py verify_gcp_leoclean_service_environment.py | sort)" actual_entries="$(sudo find "$REMOTE_RUNTIME_BIN" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" if [ "$actual_entries" != "$expected_entries" ]; then echo "Runtime directory contains an unreviewed or missing entry." >&2 return 1 fi for protected_dir in \ /usr \ /usr/local \ /usr/local/libexec \ /usr/local/libexec/livingip \ "$REMOTE_RUNTIME_BIN" \ "$gcloud_config" \ "$REMOTE_DROPIN_DIR"; do sudo test -d "$protected_dir" sudo test ! -L "$protected_dir" test "$(sudo stat -c '%U:%G' "$protected_dir")" = "root:root" sudo -n -u teleo test ! -w "$protected_dir" done for protected_file in \ "$wrapper" \ "$tool" \ "$permission_verifier" \ "$service_env_verifier" \ "$server_ca" \ "$revision" \ "$dropin"; do sudo test -f "$protected_file" sudo test ! -L "$protected_file" sudo -n -u teleo test ! -w "$protected_file" done for executable_dir in \ /usr/local/sbin \ /usr/local/bin \ /usr/sbin \ /usr/bin \ /sbin \ /bin; do sudo test -d "$executable_dir" test "$(sudo stat -Lc '%U:%G' "$executable_dir")" = "root:root" sudo -n -u teleo test ! -w "$executable_dir" done for protected_file in /bin/bash /usr/bin/curl /usr/bin/env /usr/bin/gcloud /usr/bin/psql /usr/bin/python3 /usr/bin/setpriv; do assert_trusted_executable "$protected_file" done sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin" sudo grep -E '^Environment=TELEO_(KB_MODE|GCP_PROJECT|GCP_METADATA_ONLY|CLOUDSQL_|CANONICAL_CLOUDSQL_DB)' "$dropin" sudo grep -Fx 'ReadOnlyPaths=/home/teleo' "$dropin" sudo grep -Fx "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$dropin" sudo grep -Fx "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$dropin" sudo grep -Fx 'CapabilityBoundingSet=' "$dropin" sudo grep -Fx 'AmbientCapabilities=' "$dropin" sudo grep -Fx 'NoNewPrivileges=yes' "$dropin" sudo grep -Fx 'Group=teleo' "$dropin" sudo grep -Fx 'SupplementaryGroups=teleo' "$dropin" sudo grep -Fx \ 'UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT' \ "$dropin" } assert_deployed_files() { local main_pid namespace_state source_state mount_state local home_mount_state state_mount_state workspace_mount_state capability_set ambient_capabilities assert_installed_files main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)" source_state="$(stat -Lc '%d:%i' "$REMOTE_RUNTIME_BIN")" namespace_state="$(sudo nsenter -t "$main_pid" -m -- stat -Lc '%d:%i' "$SERVICE_PROFILE_BIN")" if [ "$namespace_state" != "$source_state" ]; then echo "Service namespace does not bind the reviewed runtime directory." >&2 return 1 fi mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -T "$SERVICE_PROFILE_BIN" -n -o OPTIONS)" case ",$mount_state," in *,ro,*) ;; *) echo "Service runtime bind is not read-only." >&2 return 1 ;; esac home_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M /home/teleo -n -o OPTIONS)" state_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/state" -n -o OPTIONS)" workspace_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/workspace" -n -o OPTIONS)" case ",$home_mount_state," in *,ro,*) ;; *) echo "Service home mount is not read-only." >&2 return 1 ;; esac case ",$state_mount_state," in *,rw,*) ;; *) echo "Service state mount is not writable." >&2 return 1 ;; esac case ",$workspace_mount_state," in *,rw,*) ;; *) echo "Service workspace mount is not writable." >&2 return 1 ;; esac sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w /home sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT" sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT/profiles" sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_ROOT" sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_BIN" sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/state" sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/workspace" capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)" ambient_capabilities="$(systemctl show "$SERVICE" --property=AmbientCapabilities --value)" if [ -n "$capability_set" ] || [ -n "$ambient_capabilities" ]; then echo "Service effective capability sets are not empty." >&2 return 1 fi test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb" | awk '{print $1}')" = "$WRAPPER_SHA" test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql_memory_tool.py" | awk '{print $1}')" = "$TOOL_SHA" test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_runtime_permissions.py" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_service_environment.py" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql-server-ca.pem" | awk '{print $1}')" = "$SERVER_CA_SHA" } REMOTE ) printf -v verify_context \ 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nEXPECTED_HERMES_EXECUTABLE=%q\nEXPECTED_HERMES_PYTHON=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nSERVER_CA_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \ "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$EXPECTED_HERMES_EXECUTABLE" "$EXPECTED_HERMES_PYTHON" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$SERVER_CA_SHA" "$DROPIN_SHA" "$SOURCE_REVISION" verify_body=$(cat <<'REMOTE' set -euo pipefail verification_tmp="$(sudo mktemp -d /run/teleo-gcp-runtime-verify.XXXXXX)" sudo chown teleo:teleo "$verification_tmp" sudo chmod 0700 "$verification_tmp" cleanup_verification() { sudo rm -rf -- "$verification_tmp"; } trap cleanup_verification EXIT assert_deployed_files assert_service_running verification_service_before="$(service_fingerprint)" assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py" run_service_wrapper_status verify assert_administrator_secret_denied run_permission_verifier verify "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" "$verification_tmp/permission-receipt.json" assert_service_fingerprint "$verification_service_before" assert_service_running systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager REMOTE ) verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}" if $DRY_RUN; then echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE preflight_only=$PREFLIGHT_ONLY restart=$RESTART" echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA server_ca_sha256=$SERVER_CA_SHA dropin_sha256=$DROPIN_SHA" exit 0 fi if $VERIFY_ONLY; then "${GCP_SSH[@]}" --command="$verify_command" exit 0 fi if ! $RESTART && ! $PREFLIGHT_ONLY; then echo "Refusing to install runtime files without --restart; use --dry-run, --preflight-only, or --verify-only for read-only work." >&2 exit 64 fi printf -v sync_context \ 'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nEXPECTED_HERMES_EXECUTABLE=%q\nEXPECTED_HERMES_PYTHON=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nSERVER_CA_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nSERVER_CA_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \ "$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$EXPECTED_HERMES_EXECUTABLE" "$EXPECTED_HERMES_PYTHON" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \ "$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \ "$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$SERVER_CA_SHA" "$DROPIN_SHA" \ "$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$SERVER_CA_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY" sync_body=$(cat <<'REMOTE' set -euo pipefail remote_tmp="$(sudo mktemp -d /run/teleo-gcp-leoclean-runtime.XXXXXX)" runtime_bin="$REMOTE_RUNTIME_BIN" runtime_parent="$(dirname "$runtime_bin")" runtime_grandparent="$(dirname "$runtime_parent")" runtime_anchor="$(dirname "$runtime_grandparent")" dropin_dir="$REMOTE_DROPIN_DIR" dropin_parent="$(dirname "$dropin_dir")" payload_dir="$remote_tmp/payload" backup_dir="$remote_tmp/backup" work_dir="$remote_tmp/work" wrapper_stage="$runtime_bin/.teleo-kb.livingip-new.$$" tool_stage="$runtime_bin/.cloudsql-memory-tool.livingip-new.$$" permission_verifier_stage="$runtime_bin/.runtime-permission-verifier.livingip-new.$$" service_env_verifier_stage="$runtime_bin/.service-environment-verifier.livingip-new.$$" server_ca_stage="$runtime_bin/.cloudsql-server-ca.livingip-new.$$" revision_stage="$runtime_bin/.livingip-runtime-revision.livingip-new.$$" dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$" mutation_started=false assert_existing_root_directory() { local target="$1" if sudo test -e "$target" || sudo test -L "$target"; then sudo test -d "$target" sudo test ! -L "$target" test "$(sudo stat -c '%U:%G' "$target")" = "root:root" sudo -n -u teleo test ! -w "$target" fi } assert_existing_root_file() { local target="$1" if sudo test -e "$target" || sudo test -L "$target"; then sudo test -f "$target" sudo test ! -L "$target" test "$(sudo stat -c '%U:%G' "$target")" = "root:root" sudo -n -u teleo test ! -w "$target" fi } backup_path() { local target="$1" name="$2" if sudo test -e "$target"; then sudo cp -a -- "$target" "$backup_dir/$name" sudo touch "$backup_dir/$name.present" fi } backup_directory_state() { local target="$1" name="$2" if sudo test -d "$target"; then sudo stat -c '%u %g %a' "$target" | sudo tee "$backup_dir/$name.metadata" >/dev/null sudo touch "$backup_dir/$name.present" fi } restore_directory_state() { local target="$1" name="$2" uid gid mode if sudo test -f "$backup_dir/$name.present"; then read -r uid gid mode <<<"$(sudo cat "$backup_dir/$name.metadata")" sudo chown "$uid:$gid" "$target" sudo chmod "$mode" "$target" elif sudo test -d "$target"; then sudo rmdir -- "$target" fi } restore_path() { local target="$1" name="$2" restore_rc=0 restore_tmp if sudo test -f "$backup_dir/$name.present"; then restore_tmp="${target}.livingip-rollback.$$" sudo rm -f -- "$restore_tmp" || restore_rc=1 if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then sudo mv -f -- "$restore_tmp" "$target" || restore_rc=1 else restore_rc=1 fi sudo rm -f -- "$restore_tmp" || restore_rc=1 else sudo rm -f -- "$target" || restore_rc=1 fi return "$restore_rc" } validate_restored_path() { local target="$1" name="$2" if sudo test -f "$backup_dir/$name.present"; then sudo test -f "$target" sudo test ! -L "$target" sudo cmp -s -- "$backup_dir/$name" "$target" test "$(sudo stat -c '%u:%g:%a' "$target")" = \ "$(sudo stat -c '%u:%g:%a' "$backup_dir/$name")" else sudo test ! -e "$target" sudo test ! -L "$target" fi } validate_restored_directory() { local target="$1" name="$2" expected_state if sudo test -f "$backup_dir/$name.present"; then expected_state="$(sudo cat "$backup_dir/$name.metadata" | tr ' ' ':')" sudo test -d "$target" sudo test ! -L "$target" test "$(sudo stat -c '%u:%g:%a' "$target")" = "$expected_state" else sudo test ! -e "$target" sudo test ! -L "$target" fi } rollback_runtime() { local rollback_rc=0 echo "Deployment failed; restoring the complete previous runtime set." >&2 sudo systemctl stop "$SERVICE" || rollback_rc=1 restore_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1 restore_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 restore_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 restore_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 restore_path "$runtime_bin/cloudsql-server-ca.pem" server_ca || rollback_rc=1 restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 restore_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 restore_directory_state "$runtime_bin/gcloud-config" gcloud_config || rollback_rc=1 restore_directory_state "$runtime_bin" runtime_bin || rollback_rc=1 restore_directory_state "$runtime_parent" runtime_parent || rollback_rc=1 restore_directory_state "$runtime_grandparent" runtime_grandparent || rollback_rc=1 restore_directory_state "$dropin_dir" dropin_dir || rollback_rc=1 if [ "$rollback_rc" -eq 0 ]; then validate_restored_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1 validate_restored_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1 validate_restored_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1 validate_restored_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1 validate_restored_path "$runtime_bin/cloudsql-server-ca.pem" server_ca || rollback_rc=1 validate_restored_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1 validate_restored_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1 validate_restored_directory "$runtime_grandparent" runtime_grandparent || rollback_rc=1 validate_restored_directory "$runtime_parent" runtime_parent || rollback_rc=1 validate_restored_directory "$runtime_bin" runtime_bin || rollback_rc=1 validate_restored_directory "$runtime_bin/gcloud-config" gcloud_config || rollback_rc=1 validate_restored_directory "$dropin_dir" dropin_dir || rollback_rc=1 fi if [ "$rollback_rc" -eq 0 ]; then sudo systemctl daemon-reload || rollback_rc=1 fi if [ "$rollback_rc" -eq 0 ]; then assert_unit_configuration false || rollback_rc=1 fi if [ "$rollback_rc" -eq 0 ]; then sudo systemctl start "$SERVICE" || rollback_rc=1 assert_service_running false || rollback_rc=1 fi return "$rollback_rc" } on_exit() { local rc="$1" rollback_rc=0 trap - EXIT INT TERM if [ "$rc" -ne 0 ] && [ "$mutation_started" = true ]; then set +e rollback_runtime rollback_rc=$? set -e if [ "$rollback_rc" -ne 0 ]; then echo "CRITICAL: previous runtime restoration did not return the service to active/running." >&2 rc=1 fi fi sudo rm -f -- \ "$wrapper_stage" \ "$tool_stage" \ "$permission_verifier_stage" \ "$service_env_verifier_stage" \ "$server_ca_stage" \ "$revision_stage" \ "$dropin_stage" || true sudo rm -rf -- "$remote_tmp" || true exit "$rc" } trap 'on_exit $?' EXIT trap 'exit 130' INT trap 'exit 143' TERM sudo chmod 0711 "$remote_tmp" sudo install -d -o root -g root -m 0755 "$payload_dir" sudo install -d -o root -g root -m 0700 "$backup_dir" sudo install -d -o teleo -g teleo -m 0700 "$work_dir" sudo tar --no-same-owner --no-same-permissions --no-xattrs --no-acls --no-selinux -C "$payload_dir" -xf - if sudo find "$payload_dir" -type l -print -quit | grep -q .; then echo "Staged runtime payload contains a symlink." >&2 exit 65 fi if sudo find "$payload_dir" ! -type d ! -type f -print -quit | grep -q .; then echo "Staged runtime payload contains a non-regular entry." >&2 exit 65 fi sudo chown -R root:root "$payload_dir" sudo find "$payload_dir" -type d -exec chmod 0555 {} + sudo find "$payload_dir" -type f -exec chmod 0444 {} + sudo install -d -o root -g root -m 0555 "$payload_dir/gcloud-config" sudo chmod 0555 \ "$payload_dir/$WRAPPER_REL" \ "$payload_dir/$TOOL_REL" \ "$payload_dir/$PERMISSION_VERIFIER_REL" \ "$payload_dir/$SERVICE_ENV_VERIFIER_REL" test "$(sudo sha256sum "$payload_dir/$SERVER_CA_REL" | awk '{print $1}')" = "$SERVER_CA_SHA" test "$(sudo sha256sum "$payload_dir/$WRAPPER_REL" | awk '{print $1}')" = "$WRAPPER_SHA" test "$(sudo sha256sum "$payload_dir/$TOOL_REL" | awk '{print $1}')" = "$TOOL_SHA" test "$(sudo sha256sum "$payload_dir/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA" test "$(sudo sha256sum "$payload_dir/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA" test "$(sudo sha256sum "$payload_dir/$DROPIN_REL" | awk '{print $1}')" = "$DROPIN_SHA" test "$(sudo stat -c '%U:%G:%a' "$remote_tmp")" = "root:root:711" test "$(sudo stat -c '%U:%G:%a' "$payload_dir")" = "root:root:555" test "$(sudo stat -c '%U:%G:%a' "$backup_dir")" = "root:root:700" test "$(sudo stat -c '%U:%G:%a' "$work_dir")" = "teleo:teleo:700" sudo -n -u teleo test ! -w "$remote_tmp" sudo -n -u teleo test ! -w "$payload_dir" sudo -n -u teleo test ! -w "$backup_dir" sudo test -d "$runtime_anchor" sudo test -d "$dropin_parent" sudo test ! -L "$runtime_anchor" sudo test ! -L "$dropin_parent" test "$(sudo stat -c '%U:%G' "$runtime_anchor")" = "root:root" test "$(sudo stat -c '%U:%G' "$dropin_parent")" = "root:root" sudo -n -u teleo test ! -w "$runtime_anchor" sudo -n -u teleo test ! -w "$dropin_parent" for existing_directory in \ "$runtime_grandparent" \ "$runtime_parent" \ "$runtime_bin" \ "$runtime_bin/gcloud-config" \ "$dropin_dir"; do assert_existing_root_directory "$existing_directory" done for existing_file in \ "$runtime_bin/teleo-kb" \ "$runtime_bin/cloudsql_memory_tool.py" \ "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" \ "$runtime_bin/verify_gcp_leoclean_service_environment.py" \ "$runtime_bin/cloudsql-server-ca.pem" \ "$runtime_bin/.livingip-runtime-revision" \ "$dropin_dir/$DROPIN_NAME"; do assert_existing_root_file "$existing_file" done if sudo test -d "$runtime_bin/gcloud-config"; then sudo test -z "$(sudo find "$runtime_bin/gcloud-config" -mindepth 1 -print -quit)" fi assert_service_running false if [ "$PREFLIGHT_ONLY" = true ]; then preflight_service_before="$(service_fingerprint)" assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false run_scoped_status pre "$payload_dir/$TOOL_REL" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" assert_administrator_secret_denied run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" assert_service_fingerprint "$preflight_service_before" assert_service_running false systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager exit 0 fi backup_path "$runtime_bin/teleo-kb" wrapper backup_path "$runtime_bin/cloudsql_memory_tool.py" tool backup_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier backup_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier backup_path "$runtime_bin/cloudsql-server-ca.pem" server_ca backup_path "$dropin_dir/$DROPIN_NAME" dropin backup_path "$runtime_bin/.livingip-runtime-revision" revision backup_directory_state "$runtime_parent" runtime_parent backup_directory_state "$runtime_grandparent" runtime_grandparent backup_directory_state "$runtime_bin" runtime_bin backup_directory_state "$runtime_bin/gcloud-config" gcloud_config backup_directory_state "$dropin_dir" dropin_dir preflight_service_before="$(service_fingerprint)" assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false run_scoped_status pre "$payload_dir/$TOOL_REL" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" assert_administrator_secret_denied run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config" assert_service_fingerprint "$preflight_service_before" assert_service_running false mutation_started=true sudo systemctl stop "$SERVICE" sudo install -d -o root -g root -m 0755 "$runtime_parent" sudo install -d -o root -g root -m 0755 "$runtime_bin" sudo install -d -o root -g root -m 0555 "$runtime_bin/gcloud-config" sudo install -o root -g root -m 0644 "$payload_dir/$SERVER_CA_REL" "$server_ca_stage" sudo mv -f -- "$server_ca_stage" "$runtime_bin/cloudsql-server-ca.pem" sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage" sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage" sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" "$permission_verifier_stage" sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" "$service_env_verifier_stage" sudo mv -f -- "$wrapper_stage" "$runtime_bin/teleo-kb" sudo mv -f -- "$tool_stage" "$runtime_bin/cloudsql_memory_tool.py" sudo mv -f -- "$permission_verifier_stage" "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" sudo mv -f -- "$service_env_verifier_stage" "$runtime_bin/verify_gcp_leoclean_service_environment.py" sudo install -d -o root -g root -m 0755 "$dropin_dir" sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL" "$dropin_stage" sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME" printf '%s\n' "$SOURCE_REVISION" | sudo tee "$revision_stage" >/dev/null sudo chown root:root "$revision_stage" sudo chmod 0644 "$revision_stage" sudo mv -f -- "$revision_stage" "$runtime_bin/.livingip-runtime-revision" assert_installed_files sudo systemctl daemon-reload assert_unit_configuration sudo systemctl start "$SERVICE" assert_deployed_files assert_service_running post_service_before="$(service_fingerprint)" assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py" run_service_wrapper_status post assert_administrator_secret_denied run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" "$work_dir/permission-post/receipt.json" assert_service_fingerprint "$post_service_before" assert_service_running systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager mutation_started=false REMOTE ) sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}" COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \ "$WRAPPER_REL" \ "$TOOL_REL" \ "$PERMISSION_VERIFIER_REL" \ "$SERVICE_ENV_VERIFIER_REL" \ "$SERVER_CA_REL" \ "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command"