teleo/teleo-infrastructure
8
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1
teleo-infrastructure/deploy/sync-gcp-leoclean-runtime.sh

1430 lines
59 KiB
Bash
Executable file
Raw Blame History

#!/usr/bin/env bash
# Deploy the source-controlled Cloud SQL bridge and non-secret service settings
# to the GCP parallel leoclean runtime. This does not create database users,
# read secret values, send Telegram messages, or promote Cloud SQL.
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
PROJECT="${PROJECT:-teleo-501523}"
ZONE="${ZONE:-europe-west6-a}"
INSTANCE="${INSTANCE:-teleo-prod-1}"
SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}"
MERGED_REF="${MERGED_REF:-origin/main}"
REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"
SERVICE_PROFILE_BIN="${SERVICE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}"
SERVICE_PROFILE_ROOT="${SERVICE_PROFILE_ROOT:-/home/teleo/.hermes/profiles/leoclean}"
SERVICE_HERMES_ROOT="${SERVICE_HERMES_ROOT:-/home/teleo/.hermes}"
REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}"
EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}"
EXPECTED_HERMES_EXECUTABLE="${EXPECTED_HERMES_EXECUTABLE:-/home/teleo/.hermes/hermes-agent/venv/bin/hermes}"
EXPECTED_HERMES_PYTHON="${EXPECTED_HERMES_PYTHON:-/home/teleo/.hermes/hermes-agent/venv/bin/python3}"
WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb-gcp"
TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py"
PERMISSION_VERIFIER_REL="ops/verify_gcp_leoclean_runtime_permissions.py"
SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py"
SERVER_CA_REL="ops/gcp-teleo-pgvector-standby-server-ca.pem"
DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf"
DROPIN_NAME="10-cloudsql-memory.conf"
DRY_RUN=false
RESTART=false
VERIFY_ONLY=false
PREFLIGHT_ONLY=false
usage() {
cat <<'USAGE'
Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--preflight-only] [--restart] [--verify-only]
The database role and scoped Secret Manager secret must exist before --restart.
This script deploys only the bridge files and non-secret systemd drop-in.
USAGE
}
for arg in "$@"; do
case "$arg" in
--dry-run) DRY_RUN=true ;;
--preflight-only) PREFLIGHT_ONLY=true ;;
--restart) RESTART=true ;;
--verify-only) VERIFY_ONLY=true ;;
--help|-h)
usage
exit 0
;;
*)
echo "Unknown arg: $arg" >&2
usage >&2
exit 64
;;
esac
done
if $PREFLIGHT_ONLY && $RESTART; then
echo "--preflight-only and --restart are mutually exclusive." >&2
exit 64
fi
if $VERIFY_ONLY && { $PREFLIGHT_ONLY || $RESTART; }; then
echo "--verify-only cannot be combined with --preflight-only or --restart." >&2
exit 64
fi
for path in \
"$WRAPPER_REL" \
"$TOOL_REL" \
"$PERMISSION_VERIFIER_REL" \
"$SERVICE_ENV_VERIFIER_REL" \
"$SERVER_CA_REL" \
"$DROPIN_REL"; do
if [ ! -f "$REPO_ROOT/$path" ]; then
echo "Required source file is missing: $path" >&2
exit 66
fi
if [ -L "$REPO_ROOT/$path" ]; then
echo "Refusing symlinked runtime source file: $path" >&2
exit 65
fi
done
if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2
exit 65
fi
if ! grep -Fxq 'Environment=TELEO_GCP_METADATA_ONLY=1' "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in must enforce metadata-only GCP credentials." >&2
exit 65
fi
if ! grep -Fxq "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in does not bind the root-controlled runtime into the service profile." >&2
exit 65
fi
if ! grep -Fxq 'ReadOnlyPaths=/home/teleo' "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in does not make the complete service home read-only." >&2
exit 65
fi
if ! grep -Fxq "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in does not limit service-profile writes to state and workspace." >&2
exit 65
fi
if ! grep -Fxq 'CapabilityBoundingSet=' "$REPO_ROOT/$DROPIN_REL" ||
! grep -Fxq 'AmbientCapabilities=' "$REPO_ROOT/$DROPIN_REL" ||
! grep -Fxq 'NoNewPrivileges=yes' "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in must remove all capabilities and enforce no-new-privileges." >&2
exit 65
fi
if ! grep -Fxq 'Group=teleo' "$REPO_ROOT/$DROPIN_REL" ||
! grep -Fxq 'SupplementaryGroups=teleo' "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in must bind the exact service group set." >&2
exit 65
fi
if ! grep -Fxq 'UnsetEnvironment=OPENSSL_CONF OPENSSL_MODULES OPENSSL_ENGINES GCONV_PATH CREDENTIALS_DIRECTORY' "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in must remove startup-loader and systemd-credential injection fields." >&2
exit 65
fi
if ! grep -Fxq 'InaccessiblePaths=-/home/teleo/.config/gcloud -/home/teleo/.pgpass -/home/teleo/.pg_service.conf -/home/teleo/.postgresql' "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in must mask default GCP and PostgreSQL credential paths." >&2
exit 65
fi
if ! grep -Fxq \
'UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT' \
"$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in must unset optional Cloud SQL and memory controls." >&2
exit 65
fi
SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)"
if ! git -C "$REPO_ROOT" diff --quiet HEAD -- \
"$WRAPPER_REL" \
"$TOOL_REL" \
"$PERMISSION_VERIFIER_REL" \
"$SERVICE_ENV_VERIFIER_REL" \
"$SERVER_CA_REL" \
"$DROPIN_REL"; then
echo "Refusing to deploy runtime files that do not match HEAD." >&2
exit 65
fi
if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- \
"$WRAPPER_REL" \
"$TOOL_REL" \
"$PERMISSION_VERIFIER_REL" \
"$SERVICE_ENV_VERIFIER_REL" \
"$SERVER_CA_REL" \
"$DROPIN_REL")" ]; then
echo "Refusing to deploy untracked runtime files." >&2
exit 65
fi
if ! git -C "$REPO_ROOT" rev-parse --verify "$MERGED_REF^{commit}" >/dev/null 2>&1; then
echo "Merged reference is unavailable: $MERGED_REF" >&2
exit 69
fi
if ! git -C "$REPO_ROOT" merge-base --is-ancestor "$SOURCE_REVISION" "$MERGED_REF"; then
echo "Refusing to deploy unmerged revision $SOURCE_REVISION (not contained in $MERGED_REF)." >&2
exit 65
fi
WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')"
TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')"
PERMISSION_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$PERMISSION_VERIFIER_REL" | awk '{print $1}')"
SERVICE_ENV_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')"
SERVER_CA_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVER_CA_REL" | awk '{print $1}')"
DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')"
GCP_SSH=(
gcloud compute ssh "$INSTANCE"
--project="$PROJECT"
--zone="$ZONE"
--tunnel-through-iap
)
remote_helpers=$(cat <<'REMOTE'
assert_exact_word_set() {
local actual="$1" expected="$2" label="$3" actual_sorted expected_sorted word
actual_sorted="$(for word in $actual; do printf '%s\n' "${word#-}"; done | sort)"
expected_sorted="$(for word in $expected; do printf '%s\n' "${word#-}"; done | sort)"
if [ "$actual_sorted" != "$expected_sorted" ]; then
echo "Service effective $label differs from the reviewed exact allowlist." >&2
return 1
fi
}
assert_service_running() {
local require_reviewed="${1:-true}"
local active_state sub_state main_pid unit_user process_user teleo_uid teleo_gid
local uid_fields gid_fields groups_field exec_start process_executable
local seccomp_mode seccomp_filters lsm_context
local host_user_namespace service_user_namespace
active_state="$(systemctl show "$SERVICE" --property=ActiveState --value)"
sub_state="$(systemctl show "$SERVICE" --property=SubState --value)"
if [ "$active_state" != "active" ] || [ "$sub_state" != "running" ]; then
echo "Service is not active/running: ActiveState=$active_state SubState=$sub_state" >&2
return 1
fi
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
unit_user="$(systemctl show "$SERVICE" --property=User --value)"
case "$main_pid" in
''|0|*[!0-9]*)
echo "Service has no live MainPID: $main_pid" >&2
return 1
;;
esac
if [ "$unit_user" != "teleo" ]; then
echo "Service unit user is not teleo: $unit_user" >&2
return 1
fi
process_user="$(ps -o user= -p "$main_pid" | awk '{print $1}')"
if [ "$process_user" != "teleo" ]; then
echo "Service process user is not teleo: $process_user" >&2
return 1
fi
if [ "$require_reviewed" = true ]; then
teleo_uid="$(id -u teleo)"
teleo_gid="$(id -g teleo)"
uid_fields="$(awk '/^Uid:/{print $2 ":" $3 ":" $4 ":" $5}' "/proc/$main_pid/status")"
gid_fields="$(awk '/^Gid:/{print $2 ":" $3 ":" $4 ":" $5}' "/proc/$main_pid/status")"
groups_field="$(awk '/^Groups:/{for (i=2;i<=NF;i++) printf "%s%s", (i==2 ? "" : ":"), $i}' "/proc/$main_pid/status")"
if [ "$uid_fields" != "$teleo_uid:$teleo_uid:$teleo_uid:$teleo_uid" ] ||
[ "$gid_fields" != "$teleo_gid:$teleo_gid:$teleo_gid:$teleo_gid" ] ||
[ "$groups_field" != "$teleo_gid" ]; then
echo "Service process does not have the exact reviewed uid/gid/group set." >&2
return 1
fi
fi
exec_start="$(systemctl show "$SERVICE" --property=ExecStart --value)"
case "$exec_start" in
*"path=$EXPECTED_HERMES_EXECUTABLE ; argv[]=$EXPECTED_HERMES_EXECUTABLE -p leoclean gateway run ;"*) ;;
*)
echo "Service effective ExecStart is not the reviewed leoclean Hermes gateway." >&2
return 1
;;
esac
sudo /usr/bin/python3 -I - "$main_pid" "$EXPECTED_HERMES_PYTHON" "$EXPECTED_HERMES_EXECUTABLE" <<'PY'
import os
import sys
from pathlib import Path
pid, expected_python, expected_hermes = sys.argv[1:]
raw = Path(f"/proc/{pid}/cmdline").read_bytes()
if not raw.endswith(b"\0") or b"\0\0" in raw:
raise SystemExit("service cmdline is malformed")
try:
argv = [part.decode("utf-8", "strict") for part in raw[:-1].split(b"\0")]
except UnicodeDecodeError:
raise SystemExit("service cmdline is not valid UTF-8") from None
if argv != [expected_python, expected_hermes, "-p", "leoclean", "gateway", "run"]:
raise SystemExit("service cmdline is not the reviewed leoclean Hermes gateway")
if os.path.realpath(f"/proc/{pid}/exe") != os.path.realpath(expected_python):
raise SystemExit("service executable is not the reviewed Hermes interpreter")
PY
if [ "$require_reviewed" = true ]; then
for capability_field in CapInh CapPrm CapEff CapBnd CapAmb; do
if [ "$(awk -v key="$capability_field:" '$1 == key {print $2}' "/proc/$main_pid/status")" != 0000000000000000 ]; then
echo "Service process retains capabilities in $capability_field." >&2
return 1
fi
done
if [ "$(awk '/^NoNewPrivs:/{print $2}' "/proc/$main_pid/status")" != 1 ]; then
echo "Service process does not enforce no-new-privileges." >&2
return 1
fi
seccomp_mode="$(awk '/^Seccomp:/{print $2}' "/proc/$main_pid/status")"
seccomp_filters="$(awk '/^Seccomp_filters:/{print $2}' "/proc/$main_pid/status")"
lsm_context="$(sudo cat "/proc/$main_pid/attr/current")"
host_user_namespace="$(sudo stat -Lc '%d:%i' /proc/1/ns/user)"
service_user_namespace="$(sudo stat -Lc '%d:%i' "/proc/$main_pid/ns/user")"
if [ "$seccomp_mode" != 0 ] ||
{ [ -n "$seccomp_filters" ] && [ "$seccomp_filters" != 0 ]; } ||
[ "$lsm_context" != unconfined ] ||
[ "$service_user_namespace" != "$host_user_namespace" ]; then
echo "Service process has an unreviewed live seccomp, LSM, or user-namespace restriction." >&2
return 1
fi
fi
}
assert_unit_configuration() {
local require_reviewed="${1:-true}"
local dropin_path dropin_name dropin_paths environment_files reviewed_dropin_seen=false
local property value capability_set ambient_capabilities no_new_privileges unit_group supplementary_groups
local inaccessible_paths credential_path
local read_only_paths read_write_paths bind_paths bind_read_only_paths temporary_file_systems
local private_network private_users network_property
dropin_paths="$(systemctl show "$SERVICE" --property=DropInPaths --value)"
environment_files="$(systemctl show "$SERVICE" --property=EnvironmentFiles --value)"
if [ -n "$environment_files" ]; then
echo "Service has an effective EnvironmentFile, which is not allowed for the scoped runtime." >&2
return 1
fi
for property in LoadCredential LoadCredentialEncrypted SetCredential SetCredentialEncrypted ImportCredential ExecCondition ExecStartPre ExecStartPost ExecReload ExecStop ExecStopPost; do
value="$(systemctl show "$SERVICE" --property="$property" --value)"
if [ -n "$value" ]; then
echo "Service has an effective $property credential source." >&2
return 1
fi
done
for dropin_path in $dropin_paths; do
if [ "$dropin_path" = "$REMOTE_DROPIN_DIR/$DROPIN_NAME" ]; then
reviewed_dropin_seen=true
continue
fi
dropin_name="${dropin_path##*/}"
if [[ "$dropin_name" > "$DROPIN_NAME" ]]; then
echo "Service has a later drop-in than the reviewed Cloud SQL configuration: $dropin_name" >&2
return 1
fi
done
if [ "$require_reviewed" = true ] && [ "$reviewed_dropin_seen" != true ]; then
echo "Service does not load the reviewed Cloud SQL drop-in." >&2
return 1
fi
if [ "$require_reviewed" = true ]; then
capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)"
ambient_capabilities="$(systemctl show "$SERVICE" --property=AmbientCapabilities --value)"
no_new_privileges="$(systemctl show "$SERVICE" --property=NoNewPrivileges --value)"
unit_group="$(systemctl show "$SERVICE" --property=Group --value)"
supplementary_groups="$(systemctl show "$SERVICE" --property=SupplementaryGroups --value)"
inaccessible_paths="$(systemctl show "$SERVICE" --property=InaccessiblePaths --value)"
read_only_paths="$(systemctl show "$SERVICE" --property=ReadOnlyPaths --value)"
read_write_paths="$(systemctl show "$SERVICE" --property=ReadWritePaths --value)"
bind_paths="$(systemctl show "$SERVICE" --property=BindPaths --value)"
bind_read_only_paths="$(systemctl show "$SERVICE" --property=BindReadOnlyPaths --value)"
temporary_file_systems="$(systemctl show "$SERVICE" --property=TemporaryFileSystem --value)"
if [ -n "$capability_set" ] || [ -n "$ambient_capabilities" ] || [ "$no_new_privileges" != yes ] ||
[ "$unit_group" != teleo ] || [ "$supplementary_groups" != teleo ]; then
echo "Service effective capability, privilege, or group contract differs from the reviewed drop-in." >&2
return 1
fi
assert_exact_word_set "$read_only_paths" "/home/teleo" ReadOnlyPaths
assert_exact_word_set "$read_write_paths" "$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" ReadWritePaths
assert_exact_word_set "$bind_paths" "" BindPaths
assert_exact_word_set "$bind_read_only_paths" "$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" BindReadOnlyPaths
assert_exact_word_set "$temporary_file_systems" "" TemporaryFileSystem
assert_exact_word_set "$supplementary_groups" "teleo" SupplementaryGroups
assert_exact_word_set "$inaccessible_paths" "/home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql" InaccessiblePaths
private_network="$(systemctl show "$SERVICE" --property=PrivateNetwork --value)"
private_users="$(systemctl show "$SERVICE" --property=PrivateUsers --value)"
if [ "$private_network" != no ] || [ "$private_users" != no ]; then
echo "Service effective private network or user namespace differs from the reviewed host contract." >&2
return 1
fi
for network_property in \
NetworkNamespacePath \
IPAddressDeny \
IPAddressAllow \
RestrictAddressFamilies \
RestrictNetworkInterfaces \
SocketBindAllow \
SocketBindDeny \
SystemCallFilter \
AppArmorProfile \
SELinuxContext; do
value="$(systemctl show "$SERVICE" --property="$network_property" --value)"
if [ -n "$value" ]; then
echo "Service effective $network_property differs from the reviewed unrestricted transport contract." >&2
return 1
fi
done
for credential_path in /home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql; do
case " $inaccessible_paths " in
*" $credential_path "*) ;;
*)
echo "Service does not mask a default credential path: $credential_path" >&2
return 1
;;
esac
done
fi
}
assert_runtime_environment() {
local verifier_path="$1" require_reviewed="${2:-true}" main_pid
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
if [ "$require_reviewed" = true ]; then
sudo /usr/bin/python3 -I "$verifier_path" --pid "$main_pid"
else
sudo /usr/bin/python3 -I "$verifier_path" --pid "$main_pid" --allow-missing-runtime-security-fields
fi
assert_unit_configuration "$require_reviewed"
if [ "$require_reviewed" = true ]; then
sudo nsenter -t "$main_pid" -m -- test ! -e "/run/credentials/$SERVICE"
for credential_path in /home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql; do
sudo nsenter -t "$main_pid" -m -- \
/usr/bin/setpriv --reuid=teleo --regid=teleo --clear-groups --no-new-privs \
/usr/bin/test ! -r "$credential_path"
done
fi
}
assert_attached_service_account() {
local metadata_email
metadata_email="$(
sudo -n -u teleo env -i \
HOME=/home/teleo \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
/usr/bin/curl -q --noproxy '*' --proto '=http' --max-time 10 --fail --silent --show-error \
--header 'Metadata-Flavor: Google' \
'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email'
)"
if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then
echo "Attached VM service account mismatch: $metadata_email" >&2
return 1
fi
echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email"
}
run_in_service_cgroup() {
local main_pid="$1" control_group
shift
control_group="$(systemctl show "$SERVICE" --property=ControlGroup --value)"
if [ -z "$control_group" ]; then
echo "Service has no effective cgroup." >&2
return 1
fi
sudo /usr/bin/python3 -I - "$main_pid" "$control_group" "$@" <<'PY'
import ctypes
import os
import resource
import signal
import stat
import sys
from pathlib import Path, PurePosixPath
HANDLED_SIGNALS = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)
PR_SET_PDEATHSIG = 1
class ParentInterrupted(BaseException):
def __init__(self, signum: int) -> None:
self.signum = signum
def fail() -> None:
raise SystemExit("service-context runner failed closed")
def process_cgroup(pid: str) -> str:
try:
lines = Path(f"/proc/{pid}/cgroup").read_text(encoding="ascii").splitlines()
except (OSError, UnicodeError):
fail()
matches = []
for line in lines:
parts = line.split(":", 2)
if len(parts) == 3 and parts[0] == "0" and parts[1] == "":
matches.append(parts[2])
if len(matches) != 1:
fail()
value = matches[0]
path = PurePosixPath(value)
if not value.startswith("/") or ".." in path.parts or str(path) != value:
fail()
return value
def set_parent_death_signal(expected_parent: int) -> None:
libc = ctypes.CDLL(None, use_errno=True)
prctl = libc.prctl
prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong]
prctl.restype = ctypes.c_int
if prctl(PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) != 0:
os._exit(125)
if os.getppid() != expected_parent:
os._exit(125)
def kill_and_reap(child: int) -> None:
try:
os.kill(child, signal.SIGKILL)
except ProcessLookupError:
pass
while True:
try:
os.waitpid(child, 0)
return
except InterruptedError:
continue
except ChildProcessError:
return
def supported_resource_ids() -> tuple[int, ...]:
identifiers = set()
for name in dir(resource):
if not name.startswith("RLIMIT_") or name == "RLIMIT_NLIMITS":
continue
identifier = getattr(resource, name)
if not isinstance(identifier, int) or identifier < 0:
continue
try:
resource.prlimit(os.getpid(), identifier)
except (OSError, ValueError):
continue
identifiers.add(identifier)
return tuple(sorted(identifiers))
def copy_process_limits(source_pid: int, child: int) -> None:
snapshot = {
identifier: resource.prlimit(source_pid, identifier)
for identifier in supported_resource_ids()
}
for identifier, limits in snapshot.items():
resource.prlimit(child, identifier, limits)
if resource.prlimit(child, identifier) != limits:
fail()
if any(resource.prlimit(source_pid, identifier) != limits for identifier, limits in snapshot.items()):
fail()
def no_op_stopped_child(_child: int) -> None:
return None
def run_stopped_child(
command: list[str],
expected_cgroup: str,
*,
move_child,
read_child_cgroup,
prepare_child=set_parent_death_signal,
configure_stopped_child=no_op_stopped_child,
) -> int:
if not command or not command[0].startswith("/"):
fail()
original_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS)
parent_pid = os.getpid()
try:
child = os.fork()
except BaseException:
signal.pthread_sigmask(signal.SIG_SETMASK, original_mask)
raise
if child == 0:
try:
for signum in HANDLED_SIGNALS:
signal.signal(signum, signal.SIG_DFL)
prepare_child(parent_pid)
signal.pthread_sigmask(signal.SIG_SETMASK, original_mask)
if os.getppid() != parent_pid:
os._exit(125)
os.kill(os.getpid(), signal.SIGSTOP)
os.execv(command[0], command)
except BaseException:
os._exit(126)
previous_handlers = {}
reaped = False
interrupted = None
status = None
def interrupt(signum, _frame) -> None:
signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS)
raise ParentInterrupted(signum)
try:
for signum in HANDLED_SIGNALS:
previous_handlers[signum] = signal.signal(signum, interrupt)
signal.pthread_sigmask(signal.SIG_SETMASK, original_mask)
waited, status = os.waitpid(child, os.WUNTRACED)
if waited != child or not os.WIFSTOPPED(status) or os.WSTOPSIG(status) != signal.SIGSTOP:
fail()
move_child(child)
if read_child_cgroup(str(child)) != expected_cgroup:
fail()
configure_stopped_child(child)
os.kill(child, signal.SIGCONT)
waited, status = os.waitpid(child, 0)
if waited != child:
fail()
reaped = True
except ParentInterrupted as exc:
interrupted = exc.signum
finally:
signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS)
if not reaped:
kill_and_reap(child)
for signum, previous in previous_handlers.items():
signal.signal(signum, previous)
signal.pthread_sigmask(signal.SIG_SETMASK, original_mask)
if interrupted is not None:
signal.signal(interrupted, signal.SIG_DFL)
os.kill(os.getpid(), interrupted)
fail()
if status is None:
fail()
if os.WIFEXITED(status):
return os.WEXITSTATUS(status)
if os.WIFSIGNALED(status):
return 128 + os.WTERMSIG(status)
fail()
def main() -> int:
if len(sys.argv) < 4:
fail()
main_pid, expected_cgroup, *command = sys.argv[1:]
if not main_pid.isascii() or not main_pid.isdecimal() or int(main_pid) <= 0:
fail()
if process_cgroup(main_pid) != expected_cgroup:
fail()
cgroup_root = Path("/sys/fs/cgroup").resolve(strict=True)
cgroup_dir = (cgroup_root / expected_cgroup.lstrip("/")).resolve(strict=True)
try:
cgroup_dir.relative_to(cgroup_root)
except ValueError:
fail()
cgroup_procs = cgroup_dir / "cgroup.procs"
try:
if not stat.S_ISREG(cgroup_procs.stat().st_mode):
fail()
except OSError:
fail()
def move_child(child: int) -> None:
with cgroup_procs.open("w", encoding="ascii") as handle:
handle.write(f"{child}\n")
return run_stopped_child(
command,
expected_cgroup,
move_child=move_child,
read_child_cgroup=process_cgroup,
configure_stopped_child=lambda child: copy_process_limits(int(main_pid), child),
)
if __name__ == "__main__":
raise SystemExit(main())
PY
}
assert_administrator_secret_denied() {
local main_pid probe_program
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
probe_program="$(cat <<'PY'
import json
import os
import subprocess
import sys
import urllib.error
import urllib.parse
import urllib.request
project, expected_email = sys.argv[1:]
metadata_headers = {"Metadata-Flavor": "Google"}
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def metadata(path: str) -> bytes:
request = urllib.request.Request(
"http://169.254.169.254/computeMetadata/v1/" + path,
headers=metadata_headers,
)
with opener.open(request, timeout=10) as response:
return response.read(65536)
try:
email = metadata("instance/service-accounts/default/email").decode("utf-8").strip()
if email != expected_email:
raise RuntimeError
token_payload = json.loads(metadata("instance/service-accounts/default/token"))
token = token_payload.get("access_token")
if token_payload.get("token_type") != "Bearer" or not isinstance(token, str) or not token:
raise RuntimeError
secret = urllib.parse.quote("gcp-teleo-pgvector-standby-postgres-password", safe="")
url = f"https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{secret}/versions/latest:access"
request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
try:
response = opener.open(request, timeout=10)
except urllib.error.HTTPError as error:
body = error.read(65536)
parsed = json.loads(body)
detail = parsed.get("error", {})
message = str(detail.get("message", "")).lower()
if error.code != 403 or detail.get("status") != "PERMISSION_DENIED":
raise RuntimeError from None
if "secretmanager.versions.access" not in message:
raise RuntimeError
else:
response.close()
raise RuntimeError
gcloud_command = [
"/usr/bin/gcloud",
"secrets",
"versions",
"access",
"latest",
"--secret=gcp-teleo-pgvector-standby-postgres-password",
f"--project={project}",
"--quiet",
]
default_discovery_environment = os.environ.copy()
for key in (
"CLOUDSDK_CONFIG",
"GOOGLE_APPLICATION_CREDENTIALS",
"PGPASSFILE",
"PGSERVICE",
"XDG_CONFIG_HOME",
):
default_discovery_environment.pop(key, None)
default_discovery_environment["HOME"] = "/home/teleo"
for environment in (os.environ.copy(), default_discovery_environment):
denied = subprocess.run(
gcloud_command,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
text=True,
env=environment,
timeout=30,
check=False,
)
denial_error = denied.stderr[:65536]
if denied.returncode == 0:
raise RuntimeError
if "PERMISSION_DENIED" not in denial_error or "secretmanager.versions.access" not in denial_error:
raise RuntimeError
except Exception:
raise SystemExit("administrator secret was not the exact expected metadata-identity IAM denial") from None
print(json.dumps({
"administrator_secret_access": "iam_permission_denied",
"ambient_gcloud_administrator_secret_access": "iam_permission_denied",
"credential_context": "systemd_main_pid_environment",
"direct_metadata_administrator_secret_access": "iam_permission_denied",
"default_discovery_administrator_secret_access": "iam_permission_denied",
"identity_source": "gce_metadata_ambient_gcloud_and_default_discovery",
"stdout_discarded": True,
}, sort_keys=True))
PY
)"
run_in_service_cgroup "$main_pid" \
/usr/bin/nsenter -t "$main_pid" -m -n --env -- \
/usr/bin/setpriv \
--reuid=teleo \
--regid=teleo \
--clear-groups \
--no-new-privs \
/usr/bin/python3 -I -c "$probe_program" "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT"
}
run_scoped_status() {
local phase="$1" tool_path="$2" server_ca_path="${3:-$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem}"
local cloudsdk_config="${4:-$REMOTE_RUNTIME_BIN/gcloud-config}"
assert_attached_service_account
echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical"
sudo -n -u teleo env -i \
HOME=/home/teleo \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
TELEO_GCP_PROJECT=teleo-501523 \
TELEO_GCP_METADATA_ONLY=1 \
TELEO_CLOUDSQL_HOST=10.61.0.3 \
TELEO_CLOUDSQL_PORT=5432 \
TELEO_CLOUDSQL_DB=teleo_canonical \
TELEO_CANONICAL_CLOUDSQL_DB=teleo_canonical \
TELEO_CLOUDSQL_USER=leoclean_kb_runtime \
TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \
TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \
CLOUDSDK_CONFIG="$cloudsdk_config" \
PGSSLMODE=verify-ca \
PGSSLROOTCERT="$server_ca_path" \
TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \
TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \
/usr/bin/python3 -I "$tool_path" status --format json --redacted
}
run_service_wrapper_status() {
local phase="$1" main_pid
assert_attached_service_account
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
echo "SCOPED_WRAPPER_STATUS phase=$phase user=teleo database=teleo_canonical"
# --env imports the target MainPID's effective /proc environment. setpriv
# drops back to the service's Unix identity without sudo resetting that env.
run_in_service_cgroup "$main_pid" \
/usr/bin/nsenter -t "$main_pid" -m -n --env -- \
/usr/bin/setpriv \
--reuid=teleo \
--regid=teleo \
--clear-groups \
--no-new-privs \
"$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted
}
service_fingerprint() {
printf '%s|%s|%s|%s' \
"$(systemctl show "$SERVICE" --property=ActiveState --value)" \
"$(systemctl show "$SERVICE" --property=SubState --value)" \
"$(systemctl show "$SERVICE" --property=MainPID --value)" \
"$(systemctl show "$SERVICE" --property=NRestarts --value)"
}
assert_service_fingerprint() {
local expected="$1" actual
actual="$(service_fingerprint)"
if [ "$actual" != "$expected" ]; then
echo "Service state changed during the read-only preflight." >&2
return 1
fi
echo "PREFLIGHT_SERVICE_UNCHANGED fingerprint=$actual"
}
run_permission_verifier() {
local phase="$1" verifier_path="$2" receipt_path="$3"
local server_ca_path="${4:-$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem}" receipt_dir run_id receipt_state
local main_pid
local cloudsdk_config="${5:-$REMOTE_RUNTIME_BIN/gcloud-config}"
run_id="$(date -u +%Y%m%d%H%M%S)-$phase"
receipt_dir="$(dirname "$receipt_path")"
sudo -n -u teleo mkdir -p -- "$receipt_dir"
sudo -n -u teleo chmod 0700 "$receipt_dir"
echo "NEGATIVE_PERMISSION_RECEIPT phase=$phase run_id=$run_id"
if [ "$phase" = pre ]; then
sudo -n -u teleo env -i \
HOME=/home/teleo \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \
TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \
/usr/bin/python3 -I "$verifier_path" --run-id "$run_id" --output "$receipt_path"
else
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
run_in_service_cgroup "$main_pid" \
/usr/bin/nsenter -t "$main_pid" -m -n -- \
/usr/bin/setpriv \
--reuid=teleo \
--regid=teleo \
--clear-groups \
--no-new-privs \
/usr/bin/env -i \
HOME=/home/teleo \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
TELEO_GCP_PREFLIGHT_SSL_ROOT_CERT="$server_ca_path" \
TELEO_GCP_PREFLIGHT_CLOUDSDK_CONFIG="$cloudsdk_config" \
/usr/bin/python3 -I "$verifier_path" --run-id "$run_id" --output "$receipt_path"
fi
receipt_state="$(sudo stat -c '%U:%G:%a' "$receipt_path")"
if [ "$receipt_state" != "teleo:teleo:600" ]; then
echo "Permission receipt has unexpected ownership or mode: $receipt_state" >&2
return 1
fi
sudo /usr/bin/python3 -I - "$receipt_path" <<'PY'
import json
import sys
from pathlib import Path
receipt = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
expected = {
"artifact": "gcp_leoclean_runtime_permissions",
"current_tier": "T3_live_readonly",
"mode": "live_private_gcp_staging",
"required_tier": "T3_live_readonly",
"status": "pass",
}
mismatched = sorted(key for key, value in expected.items() if receipt.get(key) != value)
if mismatched:
raise SystemExit("permission receipt contract mismatch: " + ",".join(mismatched))
print(json.dumps({"permission_receipt": "validated", "validated_fields": sorted(expected)}, sort_keys=True))
PY
}
assert_trusted_executable() {
local requested="$1" resolved current owner
sudo test -x "$requested"
owner="$(sudo stat -c '%U' "$requested")"
if [ "$owner" != root ]; then
echo "Runtime executable path is not root-owned: $requested" >&2
return 1
fi
resolved="$(sudo readlink -f -- "$requested")"
sudo test -f "$resolved"
sudo test -x "$resolved"
current="$(dirname "$requested")"
while :; do
test "$(sudo stat -Lc '%U' "$current")" = root
sudo -n -u teleo test ! -w "$current"
[ "$current" = / ] && break
current="$(dirname "$current")"
done
current="$resolved"
while :; do
test "$(sudo stat -Lc '%U' "$current")" = root
sudo -n -u teleo test ! -w "$current"
[ "$current" = / ] && break
current="$(dirname "$current")"
done
}
assert_installed_files() {
local wrapper tool permission_verifier service_env_verifier server_ca gcloud_config dropin revision
local executable_dir protected_dir protected_file actual_entries expected_entries
wrapper="$REMOTE_RUNTIME_BIN/teleo-kb"
tool="$REMOTE_RUNTIME_BIN/cloudsql_memory_tool.py"
permission_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py"
service_env_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"
server_ca="$REMOTE_RUNTIME_BIN/cloudsql-server-ca.pem"
gcloud_config="$REMOTE_RUNTIME_BIN/gcloud-config"
dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME"
revision="$REMOTE_RUNTIME_BIN/.livingip-runtime-revision"
test "$(sudo sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA"
test "$(sudo sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA"
test "$(sudo sha256sum "$permission_verifier" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
test "$(sudo sha256sum "$service_env_verifier" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
test "$(sudo sha256sum "$server_ca" | awk '{print $1}')" = "$SERVER_CA_SHA"
test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA"
test "$(sudo stat -c '%U:%G:%a' "$REMOTE_RUNTIME_BIN")" = "root:root:755"
test "$(sudo stat -c '%U:%G:%a' "$wrapper")" = "root:root:755"
test "$(sudo stat -c '%U:%G:%a' "$tool")" = "root:root:755"
test "$(sudo stat -c '%U:%G:%a' "$permission_verifier")" = "root:root:755"
test "$(sudo stat -c '%U:%G:%a' "$service_env_verifier")" = "root:root:755"
test "$(sudo stat -c '%U:%G:%a' "$server_ca")" = "root:root:644"
test "$(sudo stat -c '%U:%G:%a' "$gcloud_config")" = "root:root:555"
test "$(sudo stat -c '%U:%G:%a' "$revision")" = "root:root:644"
test "$(sudo stat -c '%U:%G:%a' "$dropin")" = "root:root:644"
test "$(sudo cat "$revision")" = "$SOURCE_REVISION"
sudo test -z "$(sudo find "$gcloud_config" -mindepth 1 -print -quit)"
expected_entries="$(printf '%s\n' .livingip-runtime-revision cloudsql-server-ca.pem cloudsql_memory_tool.py gcloud-config teleo-kb verify_gcp_leoclean_runtime_permissions.py verify_gcp_leoclean_service_environment.py | sort)"
actual_entries="$(sudo find "$REMOTE_RUNTIME_BIN" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)"
if [ "$actual_entries" != "$expected_entries" ]; then
echo "Runtime directory contains an unreviewed or missing entry." >&2
return 1
fi
for protected_dir in \
/usr \
/usr/local \
/usr/local/libexec \
/usr/local/libexec/livingip \
"$REMOTE_RUNTIME_BIN" \
"$gcloud_config" \
"$REMOTE_DROPIN_DIR"; do
sudo test -d "$protected_dir"
sudo test ! -L "$protected_dir"
test "$(sudo stat -c '%U:%G' "$protected_dir")" = "root:root"
sudo -n -u teleo test ! -w "$protected_dir"
done
for protected_file in \
"$wrapper" \
"$tool" \
"$permission_verifier" \
"$service_env_verifier" \
"$server_ca" \
"$revision" \
"$dropin"; do
sudo test -f "$protected_file"
sudo test ! -L "$protected_file"
sudo -n -u teleo test ! -w "$protected_file"
done
for executable_dir in \
/usr/local/sbin \
/usr/local/bin \
/usr/sbin \
/usr/bin \
/sbin \
/bin; do
sudo test -d "$executable_dir"
test "$(sudo stat -Lc '%U:%G' "$executable_dir")" = "root:root"
sudo -n -u teleo test ! -w "$executable_dir"
done
for protected_file in /bin/bash /usr/bin/curl /usr/bin/env /usr/bin/gcloud /usr/bin/psql /usr/bin/python3 /usr/bin/setpriv; do
assert_trusted_executable "$protected_file"
done
sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin"
sudo grep -E '^Environment=TELEO_(KB_MODE|GCP_PROJECT|GCP_METADATA_ONLY|CLOUDSQL_|CANONICAL_CLOUDSQL_DB)' "$dropin"
sudo grep -Fx 'ReadOnlyPaths=/home/teleo' "$dropin"
sudo grep -Fx "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$dropin"
sudo grep -Fx "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$dropin"
sudo grep -Fx 'CapabilityBoundingSet=' "$dropin"
sudo grep -Fx 'AmbientCapabilities=' "$dropin"
sudo grep -Fx 'NoNewPrivileges=yes' "$dropin"
sudo grep -Fx 'Group=teleo' "$dropin"
sudo grep -Fx 'SupplementaryGroups=teleo' "$dropin"
sudo grep -Fx \
'UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT' \
"$dropin"
}
assert_deployed_files() {
local main_pid namespace_state source_state mount_state
local home_mount_state state_mount_state workspace_mount_state capability_set ambient_capabilities
assert_installed_files
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
source_state="$(stat -Lc '%d:%i' "$REMOTE_RUNTIME_BIN")"
namespace_state="$(sudo nsenter -t "$main_pid" -m -- stat -Lc '%d:%i' "$SERVICE_PROFILE_BIN")"
if [ "$namespace_state" != "$source_state" ]; then
echo "Service namespace does not bind the reviewed runtime directory." >&2
return 1
fi
mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -T "$SERVICE_PROFILE_BIN" -n -o OPTIONS)"
case ",$mount_state," in
*,ro,*) ;;
*)
echo "Service runtime bind is not read-only." >&2
return 1
;;
esac
home_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M /home/teleo -n -o OPTIONS)"
state_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/state" -n -o OPTIONS)"
workspace_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/workspace" -n -o OPTIONS)"
case ",$home_mount_state," in
*,ro,*) ;;
*)
echo "Service home mount is not read-only." >&2
return 1
;;
esac
case ",$state_mount_state," in
*,rw,*) ;;
*)
echo "Service state mount is not writable." >&2
return 1
;;
esac
case ",$workspace_mount_state," in
*,rw,*) ;;
*)
echo "Service workspace mount is not writable." >&2
return 1
;;
esac
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w /home
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT"
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT/profiles"
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_ROOT"
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_BIN"
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/state"
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/workspace"
capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)"
ambient_capabilities="$(systemctl show "$SERVICE" --property=AmbientCapabilities --value)"
if [ -n "$capability_set" ] || [ -n "$ambient_capabilities" ]; then
echo "Service effective capability sets are not empty." >&2
return 1
fi
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb" | awk '{print $1}')" = "$WRAPPER_SHA"
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql_memory_tool.py" | awk '{print $1}')" = "$TOOL_SHA"
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_runtime_permissions.py" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_service_environment.py" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql-server-ca.pem" | awk '{print $1}')" = "$SERVER_CA_SHA"
}
REMOTE
)
printf -v verify_context \
'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nEXPECTED_HERMES_EXECUTABLE=%q\nEXPECTED_HERMES_PYTHON=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nSERVER_CA_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \
"$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$EXPECTED_HERMES_EXECUTABLE" "$EXPECTED_HERMES_PYTHON" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \
"$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \
"$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$SERVER_CA_SHA" "$DROPIN_SHA" "$SOURCE_REVISION"
verify_body=$(cat <<'REMOTE'
set -euo pipefail
verification_tmp="$(sudo mktemp -d /run/teleo-gcp-runtime-verify.XXXXXX)"
sudo chown teleo:teleo "$verification_tmp"
sudo chmod 0700 "$verification_tmp"
cleanup_verification() { sudo rm -rf -- "$verification_tmp"; }
trap cleanup_verification EXIT
assert_deployed_files
assert_service_running
verification_service_before="$(service_fingerprint)"
assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"
run_service_wrapper_status verify
assert_administrator_secret_denied
run_permission_verifier verify "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" "$verification_tmp/permission-receipt.json"
assert_service_fingerprint "$verification_service_before"
assert_service_running
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
REMOTE
)
verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}"
if $DRY_RUN; then
echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE preflight_only=$PREFLIGHT_ONLY restart=$RESTART"
echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA server_ca_sha256=$SERVER_CA_SHA dropin_sha256=$DROPIN_SHA"
exit 0
fi
if $VERIFY_ONLY; then
"${GCP_SSH[@]}" --command="$verify_command"
exit 0
fi
if ! $RESTART && ! $PREFLIGHT_ONLY; then
echo "Refusing to install runtime files without --restart; use --dry-run, --preflight-only, or --verify-only for read-only work." >&2
exit 64
fi
printf -v sync_context \
'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nEXPECTED_HERMES_EXECUTABLE=%q\nEXPECTED_HERMES_PYTHON=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nSERVER_CA_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nSERVER_CA_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \
"$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$EXPECTED_HERMES_EXECUTABLE" "$EXPECTED_HERMES_PYTHON" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \
"$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \
"$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$SERVER_CA_SHA" "$DROPIN_SHA" \
"$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$SERVER_CA_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY"
sync_body=$(cat <<'REMOTE'
set -euo pipefail
remote_tmp="$(sudo mktemp -d /run/teleo-gcp-leoclean-runtime.XXXXXX)"
runtime_bin="$REMOTE_RUNTIME_BIN"
runtime_parent="$(dirname "$runtime_bin")"
runtime_grandparent="$(dirname "$runtime_parent")"
runtime_anchor="$(dirname "$runtime_grandparent")"
dropin_dir="$REMOTE_DROPIN_DIR"
dropin_parent="$(dirname "$dropin_dir")"
payload_dir="$remote_tmp/payload"
backup_dir="$remote_tmp/backup"
work_dir="$remote_tmp/work"
wrapper_stage="$runtime_bin/.teleo-kb.livingip-new.$$"
tool_stage="$runtime_bin/.cloudsql-memory-tool.livingip-new.$$"
permission_verifier_stage="$runtime_bin/.runtime-permission-verifier.livingip-new.$$"
service_env_verifier_stage="$runtime_bin/.service-environment-verifier.livingip-new.$$"
server_ca_stage="$runtime_bin/.cloudsql-server-ca.livingip-new.$$"
revision_stage="$runtime_bin/.livingip-runtime-revision.livingip-new.$$"
dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$"
mutation_started=false
assert_existing_root_directory() {
local target="$1"
if sudo test -e "$target" || sudo test -L "$target"; then
sudo test -d "$target"
sudo test ! -L "$target"
test "$(sudo stat -c '%U:%G' "$target")" = "root:root"
sudo -n -u teleo test ! -w "$target"
fi
}
assert_existing_root_file() {
local target="$1"
if sudo test -e "$target" || sudo test -L "$target"; then
sudo test -f "$target"
sudo test ! -L "$target"
test "$(sudo stat -c '%U:%G' "$target")" = "root:root"
sudo -n -u teleo test ! -w "$target"
fi
}
backup_path() {
local target="$1" name="$2"
if sudo test -e "$target"; then
sudo cp -a -- "$target" "$backup_dir/$name"
sudo touch "$backup_dir/$name.present"
fi
}
backup_directory_state() {
local target="$1" name="$2"
if sudo test -d "$target"; then
sudo stat -c '%u %g %a' "$target" | sudo tee "$backup_dir/$name.metadata" >/dev/null
sudo touch "$backup_dir/$name.present"
fi
}
restore_directory_state() {
local target="$1" name="$2" uid gid mode
if sudo test -f "$backup_dir/$name.present"; then
read -r uid gid mode <<<"$(sudo cat "$backup_dir/$name.metadata")"
sudo chown "$uid:$gid" "$target"
sudo chmod "$mode" "$target"
elif sudo test -d "$target"; then
sudo rmdir -- "$target"
fi
}
restore_path() {
local target="$1" name="$2" restore_rc=0 restore_tmp
if sudo test -f "$backup_dir/$name.present"; then
restore_tmp="${target}.livingip-rollback.$$"
sudo rm -f -- "$restore_tmp" || restore_rc=1
if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then
sudo mv -f -- "$restore_tmp" "$target" || restore_rc=1
else
restore_rc=1
fi
sudo rm -f -- "$restore_tmp" || restore_rc=1
else
sudo rm -f -- "$target" || restore_rc=1
fi
return "$restore_rc"
}
validate_restored_path() {
local target="$1" name="$2"
if sudo test -f "$backup_dir/$name.present"; then
sudo test -f "$target"
sudo test ! -L "$target"
sudo cmp -s -- "$backup_dir/$name" "$target"
test "$(sudo stat -c '%u:%g:%a' "$target")" = \
"$(sudo stat -c '%u:%g:%a' "$backup_dir/$name")"
else
sudo test ! -e "$target"
sudo test ! -L "$target"
fi
}
validate_restored_directory() {
local target="$1" name="$2" expected_state
if sudo test -f "$backup_dir/$name.present"; then
expected_state="$(sudo cat "$backup_dir/$name.metadata" | tr ' ' ':')"
sudo test -d "$target"
sudo test ! -L "$target"
test "$(sudo stat -c '%u:%g:%a' "$target")" = "$expected_state"
else
sudo test ! -e "$target"
sudo test ! -L "$target"
fi
}
rollback_runtime() {
local rollback_rc=0
echo "Deployment failed; restoring the complete previous runtime set." >&2
sudo systemctl stop "$SERVICE" || rollback_rc=1
restore_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1
restore_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
restore_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1
restore_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1
restore_path "$runtime_bin/cloudsql-server-ca.pem" server_ca || rollback_rc=1
restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1
restore_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1
restore_directory_state "$runtime_bin/gcloud-config" gcloud_config || rollback_rc=1
restore_directory_state "$runtime_bin" runtime_bin || rollback_rc=1
restore_directory_state "$runtime_parent" runtime_parent || rollback_rc=1
restore_directory_state "$runtime_grandparent" runtime_grandparent || rollback_rc=1
restore_directory_state "$dropin_dir" dropin_dir || rollback_rc=1
if [ "$rollback_rc" -eq 0 ]; then
validate_restored_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1
validate_restored_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
validate_restored_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1
validate_restored_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1
validate_restored_path "$runtime_bin/cloudsql-server-ca.pem" server_ca || rollback_rc=1
validate_restored_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1
validate_restored_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1
validate_restored_directory "$runtime_grandparent" runtime_grandparent || rollback_rc=1
validate_restored_directory "$runtime_parent" runtime_parent || rollback_rc=1
validate_restored_directory "$runtime_bin" runtime_bin || rollback_rc=1
validate_restored_directory "$runtime_bin/gcloud-config" gcloud_config || rollback_rc=1
validate_restored_directory "$dropin_dir" dropin_dir || rollback_rc=1
fi
if [ "$rollback_rc" -eq 0 ]; then
sudo systemctl daemon-reload || rollback_rc=1
fi
if [ "$rollback_rc" -eq 0 ]; then
assert_unit_configuration false || rollback_rc=1
fi
if [ "$rollback_rc" -eq 0 ]; then
sudo systemctl start "$SERVICE" || rollback_rc=1
assert_service_running false || rollback_rc=1
fi
return "$rollback_rc"
}
on_exit() {
local rc="$1" rollback_rc=0
trap - EXIT INT TERM
if [ "$rc" -ne 0 ] && [ "$mutation_started" = true ]; then
set +e
rollback_runtime
rollback_rc=$?
set -e
if [ "$rollback_rc" -ne 0 ]; then
echo "CRITICAL: previous runtime restoration did not return the service to active/running." >&2
rc=1
fi
fi
sudo rm -f -- \
"$wrapper_stage" \
"$tool_stage" \
"$permission_verifier_stage" \
"$service_env_verifier_stage" \
"$server_ca_stage" \
"$revision_stage" \
"$dropin_stage" || true
sudo rm -rf -- "$remote_tmp" || true
exit "$rc"
}
trap 'on_exit $?' EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
sudo chmod 0711 "$remote_tmp"
sudo install -d -o root -g root -m 0755 "$payload_dir"
sudo install -d -o root -g root -m 0700 "$backup_dir"
sudo install -d -o teleo -g teleo -m 0700 "$work_dir"
sudo tar --no-same-owner --no-same-permissions --no-xattrs --no-acls --no-selinux -C "$payload_dir" -xf -
if sudo find "$payload_dir" -type l -print -quit | grep -q .; then
echo "Staged runtime payload contains a symlink." >&2
exit 65
fi
if sudo find "$payload_dir" ! -type d ! -type f -print -quit | grep -q .; then
echo "Staged runtime payload contains a non-regular entry." >&2
exit 65
fi
sudo chown -R root:root "$payload_dir"
sudo find "$payload_dir" -type d -exec chmod 0555 {} +
sudo find "$payload_dir" -type f -exec chmod 0444 {} +
sudo install -d -o root -g root -m 0555 "$payload_dir/gcloud-config"
sudo chmod 0555 \
"$payload_dir/$WRAPPER_REL" \
"$payload_dir/$TOOL_REL" \
"$payload_dir/$PERMISSION_VERIFIER_REL" \
"$payload_dir/$SERVICE_ENV_VERIFIER_REL"
test "$(sudo sha256sum "$payload_dir/$SERVER_CA_REL" | awk '{print $1}')" = "$SERVER_CA_SHA"
test "$(sudo sha256sum "$payload_dir/$WRAPPER_REL" | awk '{print $1}')" = "$WRAPPER_SHA"
test "$(sudo sha256sum "$payload_dir/$TOOL_REL" | awk '{print $1}')" = "$TOOL_SHA"
test "$(sudo sha256sum "$payload_dir/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
test "$(sudo sha256sum "$payload_dir/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
test "$(sudo sha256sum "$payload_dir/$DROPIN_REL" | awk '{print $1}')" = "$DROPIN_SHA"
test "$(sudo stat -c '%U:%G:%a' "$remote_tmp")" = "root:root:711"
test "$(sudo stat -c '%U:%G:%a' "$payload_dir")" = "root:root:555"
test "$(sudo stat -c '%U:%G:%a' "$backup_dir")" = "root:root:700"
test "$(sudo stat -c '%U:%G:%a' "$work_dir")" = "teleo:teleo:700"
sudo -n -u teleo test ! -w "$remote_tmp"
sudo -n -u teleo test ! -w "$payload_dir"
sudo -n -u teleo test ! -w "$backup_dir"
sudo test -d "$runtime_anchor"
sudo test -d "$dropin_parent"
sudo test ! -L "$runtime_anchor"
sudo test ! -L "$dropin_parent"
test "$(sudo stat -c '%U:%G' "$runtime_anchor")" = "root:root"
test "$(sudo stat -c '%U:%G' "$dropin_parent")" = "root:root"
sudo -n -u teleo test ! -w "$runtime_anchor"
sudo -n -u teleo test ! -w "$dropin_parent"
for existing_directory in \
"$runtime_grandparent" \
"$runtime_parent" \
"$runtime_bin" \
"$runtime_bin/gcloud-config" \
"$dropin_dir"; do
assert_existing_root_directory "$existing_directory"
done
for existing_file in \
"$runtime_bin/teleo-kb" \
"$runtime_bin/cloudsql_memory_tool.py" \
"$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" \
"$runtime_bin/verify_gcp_leoclean_service_environment.py" \
"$runtime_bin/cloudsql-server-ca.pem" \
"$runtime_bin/.livingip-runtime-revision" \
"$dropin_dir/$DROPIN_NAME"; do
assert_existing_root_file "$existing_file"
done
if sudo test -d "$runtime_bin/gcloud-config"; then
sudo test -z "$(sudo find "$runtime_bin/gcloud-config" -mindepth 1 -print -quit)"
fi
assert_service_running false
if [ "$PREFLIGHT_ONLY" = true ]; then
preflight_service_before="$(service_fingerprint)"
assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false
run_scoped_status pre "$payload_dir/$TOOL_REL" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config"
assert_administrator_secret_denied
run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config"
assert_service_fingerprint "$preflight_service_before"
assert_service_running false
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
exit 0
fi
backup_path "$runtime_bin/teleo-kb" wrapper
backup_path "$runtime_bin/cloudsql_memory_tool.py" tool
backup_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier
backup_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier
backup_path "$runtime_bin/cloudsql-server-ca.pem" server_ca
backup_path "$dropin_dir/$DROPIN_NAME" dropin
backup_path "$runtime_bin/.livingip-runtime-revision" revision
backup_directory_state "$runtime_parent" runtime_parent
backup_directory_state "$runtime_grandparent" runtime_grandparent
backup_directory_state "$runtime_bin" runtime_bin
backup_directory_state "$runtime_bin/gcloud-config" gcloud_config
backup_directory_state "$dropin_dir" dropin_dir
preflight_service_before="$(service_fingerprint)"
assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false
run_scoped_status pre "$payload_dir/$TOOL_REL" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config"
assert_administrator_secret_denied
run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json" "$payload_dir/$SERVER_CA_REL" "$payload_dir/gcloud-config"
assert_service_fingerprint "$preflight_service_before"
assert_service_running false
mutation_started=true
sudo systemctl stop "$SERVICE"
sudo install -d -o root -g root -m 0755 "$runtime_parent"
sudo install -d -o root -g root -m 0755 "$runtime_bin"
sudo install -d -o root -g root -m 0555 "$runtime_bin/gcloud-config"
sudo install -o root -g root -m 0644 "$payload_dir/$SERVER_CA_REL" "$server_ca_stage"
sudo mv -f -- "$server_ca_stage" "$runtime_bin/cloudsql-server-ca.pem"
sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage"
sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage"
sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" "$permission_verifier_stage"
sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" "$service_env_verifier_stage"
sudo mv -f -- "$wrapper_stage" "$runtime_bin/teleo-kb"
sudo mv -f -- "$tool_stage" "$runtime_bin/cloudsql_memory_tool.py"
sudo mv -f -- "$permission_verifier_stage" "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py"
sudo mv -f -- "$service_env_verifier_stage" "$runtime_bin/verify_gcp_leoclean_service_environment.py"
sudo install -d -o root -g root -m 0755 "$dropin_dir"
sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL" "$dropin_stage"
sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME"
printf '%s\n' "$SOURCE_REVISION" | sudo tee "$revision_stage" >/dev/null
sudo chown root:root "$revision_stage"
sudo chmod 0644 "$revision_stage"
sudo mv -f -- "$revision_stage" "$runtime_bin/.livingip-runtime-revision"
assert_installed_files
sudo systemctl daemon-reload
assert_unit_configuration
sudo systemctl start "$SERVICE"
assert_deployed_files
assert_service_running
post_service_before="$(service_fingerprint)"
assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py"
run_service_wrapper_status post
assert_administrator_secret_denied
run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" "$work_dir/permission-post/receipt.json"
assert_service_fingerprint "$post_service_before"
assert_service_running
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
mutation_started=false
REMOTE
)
sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}"
COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \
"$WRAPPER_REL" \
"$TOOL_REL" \
"$PERMISSION_VERIFIER_REL" \
"$SERVICE_ENV_VERIFIER_REL" \
"$SERVER_CA_REL" \
"$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command"
Reference in a new issue View git blame Copy permalink