fix: enforce scoped GCP Leo runtime credentials

This commit is contained in:
twentyOne2x 2026-07-15 04:40:26 +02:00
parent 68625b8afa
commit b7bdcea46a
14 changed files with 5031 additions and 224 deletions

View file

@ -52,6 +52,8 @@ jobs:
ops/restore_gcp_generated_postgres_snapshot.py \
ops/sqlite_to_postgres_dump.py \
ops/verify_gcp_cloudsql_restore_readback.py \
ops/verify_gcp_leoclean_runtime_permissions.py \
ops/verify_gcp_leoclean_service_environment.py \
ops/verify_postgres_parity_manifest.py \
telegram/approvals.py \
hermes-agent/leoclean-bin/kb_tool.py \
@ -80,6 +82,8 @@ jobs:
tests/test_capture_vps_canonical_postgres_snapshot.py \
tests/test_gcp_infra_execute_canary.py \
tests/test_gcp_infra_readiness_checker.py \
tests/test_gcp_leoclean_runtime_permissions.py \
tests/test_gcp_leoclean_service_environment.py \
tests/test_gcp_runtime_baseline_apply.py \
tests/test_gcp_service_communications.py \
tests/test_gcp_cloudsql_restore_drill.py \
@ -102,12 +106,15 @@ jobs:
tests/test_restore_gcp_generated_postgres_snapshot.py \
tests/test_sqlite_to_postgres_dump.py \
tests/test_sqlite_postgres_restore_canary_capsule.py \
tests/test_teleo_agent_systemd.py \
tests/test_eval_parse.py \
tests/test_contributor.py \
tests/test_search.py
- name: Shell syntax
run: |
bash -n \
deploy/sync-gcp-leoclean-runtime.sh \
hermes-agent/leoclean-bin/teleo-kb \
ops/backup_vps_sqlite_kb.sh \
ops/run_gcp_cloudsql_restore_drill.sh \
ops/run_sqlite_postgres_restore_canary.sh

View file

@ -10,23 +10,28 @@ ZONE="${ZONE:-europe-west6-a}"
INSTANCE="${INSTANCE:-teleo-prod-1}"
SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}"
MERGED_REF="${MERGED_REF:-origin/main}"
REMOTE_PROFILE_BIN="${REMOTE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}"
REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"
SERVICE_PROFILE_BIN="${SERVICE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}"
SERVICE_PROFILE_ROOT="${SERVICE_PROFILE_ROOT:-/home/teleo/.hermes/profiles/leoclean}"
SERVICE_HERMES_ROOT="${SERVICE_HERMES_ROOT:-/home/teleo/.hermes}"
REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}"
REMOTE_CLOUDSDK_CONFIG="${REMOTE_CLOUDSDK_CONFIG:-/run/leoclean-gcp-prod-parallel}"
EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}"
WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb"
TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py"
PERMISSION_VERIFIER_REL="ops/verify_gcp_leoclean_runtime_permissions.py"
SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py"
DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf"
DROPIN_NAME="10-cloudsql-memory.conf"
DRY_RUN=false
RESTART=false
VERIFY_ONLY=false
PREFLIGHT_ONLY=false
usage() {
cat <<'USAGE'
Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--restart] [--verify-only]
Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--preflight-only] [--restart] [--verify-only]
The database role and scoped Secret Manager secret must exist before --restart.
This script deploys only the bridge files and non-secret systemd drop-in.
@ -36,6 +41,7 @@ USAGE
for arg in "$@"; do
case "$arg" in
--dry-run) DRY_RUN=true ;;
--preflight-only) PREFLIGHT_ONLY=true ;;
--restart) RESTART=true ;;
--verify-only) VERIFY_ONLY=true ;;
--help|-h)
@ -50,28 +56,72 @@ for arg in "$@"; do
esac
done
for path in "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; do
if $PREFLIGHT_ONLY && $RESTART; then
echo "--preflight-only and --restart are mutually exclusive." >&2
exit 64
fi
if $VERIFY_ONLY && { $PREFLIGHT_ONLY || $RESTART; }; then
echo "--verify-only cannot be combined with --preflight-only or --restart." >&2
exit 64
fi
for path in \
"$WRAPPER_REL" \
"$TOOL_REL" \
"$PERMISSION_VERIFIER_REL" \
"$SERVICE_ENV_VERIFIER_REL" \
"$DROPIN_REL"; do
if [ ! -f "$REPO_ROOT/$path" ]; then
echo "Required source file is missing: $path" >&2
exit 66
fi
if [ -L "$REPO_ROOT/$path" ]; then
echo "Refusing symlinked runtime source file: $path" >&2
exit 65
fi
done
if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2
exit 65
fi
if ! grep -Fxq "Environment=CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG" "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in CLOUDSDK_CONFIG does not match $REMOTE_CLOUDSDK_CONFIG." >&2
if ! grep -Fxq 'Environment=TELEO_GCP_METADATA_ONLY=1' "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in must enforce metadata-only GCP credentials." >&2
exit 65
fi
if ! grep -Fxq "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in does not bind the root-controlled runtime into the service profile." >&2
exit 65
fi
if ! grep -Fxq "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in does not protect the service-profile ancestor from replacement." >&2
exit 65
fi
if ! grep -Fxq "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in does not limit service-profile writes to state and workspace." >&2
exit 65
fi
if ! grep -Fxq 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$REPO_ROOT/$DROPIN_REL"; then
echo "Runtime drop-in must remove CAP_SYS_ADMIN from the service." >&2
exit 65
fi
SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)"
if ! git -C "$REPO_ROOT" diff --quiet HEAD -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; then
if ! git -C "$REPO_ROOT" diff --quiet HEAD -- \
"$WRAPPER_REL" \
"$TOOL_REL" \
"$PERMISSION_VERIFIER_REL" \
"$SERVICE_ENV_VERIFIER_REL" \
"$DROPIN_REL"; then
echo "Refusing to deploy runtime files that do not match HEAD." >&2
exit 65
fi
if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL")" ]; then
if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- \
"$WRAPPER_REL" \
"$TOOL_REL" \
"$PERMISSION_VERIFIER_REL" \
"$SERVICE_ENV_VERIFIER_REL" \
"$DROPIN_REL")" ]; then
echo "Refusing to deploy untracked runtime files." >&2
exit 65
fi
@ -85,6 +135,8 @@ if ! git -C "$REPO_ROOT" merge-base --is-ancestor "$SOURCE_REVISION" "$MERGED_RE
fi
WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')"
TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')"
PERMISSION_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$PERMISSION_VERIFIER_REL" | awk '{print $1}')"
SERVICE_ENV_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')"
DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')"
GCP_SSH=(
@ -123,64 +175,132 @@ assert_service_running() {
fi
}
assert_runtime_environment() {
local environment runtime_dir_state
environment="$(systemctl show "$SERVICE" --property=Environment --value)"
case " $environment " in
*" PGPASSWORD="*)
echo "Service environment still contains PGPASSWORD." >&2
assert_unit_configuration() {
local require_reviewed="${1:-true}"
local dropin_path dropin_name dropin_paths environment_files reviewed_dropin_seen=false
dropin_paths="$(systemctl show "$SERVICE" --property=DropInPaths --value)"
environment_files="$(systemctl show "$SERVICE" --property=EnvironmentFiles --value)"
if [ -n "$environment_files" ]; then
echo "Service has an effective EnvironmentFile, which is not allowed for the scoped runtime." >&2
return 1
fi
for dropin_path in $dropin_paths; do
if [ "$dropin_path" = "$REMOTE_DROPIN_DIR/$DROPIN_NAME" ]; then
reviewed_dropin_seen=true
continue
fi
dropin_name="${dropin_path##*/}"
if [[ "$dropin_name" > "$DROPIN_NAME" ]]; then
echo "Service has a later drop-in than the reviewed Cloud SQL configuration: $dropin_name" >&2
return 1
;;
esac
case " $environment " in
*" CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG "*) ;;
*)
echo "Service does not use the isolated CLOUDSDK_CONFIG." >&2
return 1
;;
esac
runtime_dir_state="$(stat -c '%U:%G:%a' "$REMOTE_CLOUDSDK_CONFIG")"
if [ "$runtime_dir_state" != "teleo:teleo:700" ]; then
echo "Isolated CLOUDSDK_CONFIG has unexpected ownership or mode: $runtime_dir_state" >&2
fi
done
if [ "$require_reviewed" = true ] && [ "$reviewed_dropin_seen" != true ]; then
echo "Service does not load the reviewed Cloud SQL drop-in." >&2
return 1
fi
}
assert_runtime_environment() {
local verifier_path="$1" require_reviewed="${2:-true}" main_pid
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
sudo /usr/bin/python3 "$verifier_path" --pid "$main_pid"
assert_unit_configuration "$require_reviewed"
}
assert_attached_service_account() {
local sdk_config="$1" metadata_email
sudo install -d -o teleo -g teleo -m 0700 "$sdk_config"
local metadata_email
metadata_email="$(
sudo -n -u teleo env -u PGPASSWORD \
sudo -n -u teleo env -i \
HOME=/home/teleo \
CLOUDSDK_CONFIG="$sdk_config" \
curl --fail --silent --show-error \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
/usr/bin/curl -q --noproxy '*' --proto '=http' --max-time 10 --fail --silent --show-error \
--header 'Metadata-Flavor: Google' \
'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email'
'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email'
)"
if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then
echo "Attached VM service account mismatch: $metadata_email" >&2
return 1
fi
# An empty, phase-local Cloud SDK config forces this token request to use
# the VM's attached identity instead of an operator's cached credentials.
sudo -n -u teleo env -u PGPASSWORD \
HOME=/home/teleo \
CLOUDSDK_CONFIG="$sdk_config" \
CLOUDSDK_CORE_PROJECT="$PROJECT" \
gcloud --quiet auth print-access-token >/dev/null
echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email"
}
assert_administrator_secret_denied() {
local main_pid
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
sudo nsenter -t "$main_pid" -m --env -- \
/usr/bin/setpriv \
--reuid=teleo \
--regid=teleo \
--init-groups \
--no-new-privs \
/usr/bin/python3 - "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" <<'PY'
import json
import sys
import urllib.error
import urllib.parse
import urllib.request
project, expected_email = sys.argv[1:]
metadata_headers = {"Metadata-Flavor": "Google"}
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def metadata(path: str) -> bytes:
request = urllib.request.Request(
"http://169.254.169.254/computeMetadata/v1/" + path,
headers=metadata_headers,
)
with opener.open(request, timeout=10) as response:
return response.read(65536)
try:
email = metadata("instance/service-accounts/default/email").decode("utf-8").strip()
if email != expected_email:
raise RuntimeError
token_payload = json.loads(metadata("instance/service-accounts/default/token"))
token = token_payload.get("access_token")
if token_payload.get("token_type") != "Bearer" or not isinstance(token, str) or not token:
raise RuntimeError
secret = urllib.parse.quote("gcp-teleo-pgvector-standby-postgres-password", safe="")
url = f"https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{secret}/versions/latest:access"
request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
try:
response = opener.open(request, timeout=10)
except urllib.error.HTTPError as error:
body = error.read(65536)
parsed = json.loads(body)
detail = parsed.get("error", {})
message = str(detail.get("message", "")).lower()
if error.code != 403 or detail.get("status") != "PERMISSION_DENIED":
raise RuntimeError from None
if "secretmanager.versions.access" not in message:
raise RuntimeError
else:
response.close()
raise RuntimeError
except Exception:
raise SystemExit("administrator secret was not the exact expected metadata-identity IAM denial") from None
print(json.dumps({
"administrator_secret_access": "iam_permission_denied",
"credential_context": "systemd_main_pid_environment",
"identity_source": "gce_metadata",
"stdout_discarded": True,
}, sort_keys=True))
PY
}
run_scoped_status() {
local phase="$1" tool_path="$2" sdk_config="$3"
assert_attached_service_account "$sdk_config"
local phase="$1" tool_path="$2"
assert_attached_service_account
echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical"
sudo -n -u teleo env -u PGPASSWORD \
sudo -n -u teleo env -i \
HOME=/home/teleo \
CLOUDSDK_CONFIG="$sdk_config" \
CLOUDSDK_CORE_PROJECT="$PROJECT" \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
TELEO_GCP_PROJECT=teleo-501523 \
TELEO_GCP_METADATA_ONLY=1 \
TELEO_CLOUDSQL_HOST=10.61.0.3 \
TELEO_CLOUDSQL_PORT=5432 \
TELEO_CLOUDSQL_DB=teleo_canonical \
@ -188,46 +308,238 @@ run_scoped_status() {
TELEO_CLOUDSQL_USER=leoclean_kb_runtime \
TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \
TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \
python3 "$tool_path" status --format json --redacted
/usr/bin/python3 "$tool_path" status --format json --redacted
}
run_service_wrapper_status() {
local phase="$1" main_pid
assert_attached_service_account
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
echo "SCOPED_WRAPPER_STATUS phase=$phase user=teleo database=teleo_canonical"
# --env imports the target MainPID's effective /proc environment. setpriv
# drops back to the service's Unix identity without sudo resetting that env.
sudo nsenter -t "$main_pid" -m --env -- \
/usr/bin/setpriv \
--reuid=teleo \
--regid=teleo \
--init-groups \
--no-new-privs \
"$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted
}
service_fingerprint() {
printf '%s|%s|%s|%s' \
"$(systemctl show "$SERVICE" --property=ActiveState --value)" \
"$(systemctl show "$SERVICE" --property=SubState --value)" \
"$(systemctl show "$SERVICE" --property=MainPID --value)" \
"$(systemctl show "$SERVICE" --property=NRestarts --value)"
}
assert_service_fingerprint() {
local expected="$1" actual
actual="$(service_fingerprint)"
if [ "$actual" != "$expected" ]; then
echo "Service state changed during the read-only preflight." >&2
return 1
fi
echo "PREFLIGHT_SERVICE_UNCHANGED fingerprint=$actual"
}
run_permission_verifier() {
local phase="$1" verifier_path="$2" receipt_path="$3" receipt_dir run_id receipt_state
run_id="$(date -u +%Y%m%d%H%M%S)-$phase"
receipt_dir="$(dirname "$receipt_path")"
sudo -n -u teleo mkdir -p -- "$receipt_dir"
sudo -n -u teleo chmod 0700 "$receipt_dir"
echo "NEGATIVE_PERMISSION_RECEIPT phase=$phase run_id=$run_id"
sudo -n -u teleo env -i \
HOME=/home/teleo \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
/usr/bin/python3 "$verifier_path" --run-id "$run_id" --output "$receipt_path"
receipt_state="$(sudo stat -c '%U:%G:%a' "$receipt_path")"
if [ "$receipt_state" != "teleo:teleo:600" ]; then
echo "Permission receipt has unexpected ownership or mode: $receipt_state" >&2
return 1
fi
sudo /usr/bin/python3 - "$receipt_path" <<'PY'
import json
import sys
from pathlib import Path
receipt = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
expected = {
"artifact": "gcp_leoclean_runtime_permissions",
"current_tier": "T3_live_readonly",
"mode": "live_private_gcp_staging",
"required_tier": "T3_live_readonly",
"status": "pass",
}
mismatched = sorted(key for key, value in expected.items() if receipt.get(key) != value)
if mismatched:
raise SystemExit("permission receipt contract mismatch: " + ",".join(mismatched))
print(json.dumps({"permission_receipt": "validated", "validated_fields": sorted(expected)}, sort_keys=True))
PY
}
assert_installed_files() {
local wrapper tool permission_verifier service_env_verifier dropin revision executable_dir protected_dir protected_file
wrapper="$REMOTE_RUNTIME_BIN/teleo-kb"
tool="$REMOTE_RUNTIME_BIN/cloudsql_memory_tool.py"
permission_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py"
service_env_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"
dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME"
revision="$REMOTE_RUNTIME_BIN/.livingip-runtime-revision"
test "$(sudo sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA"
test "$(sudo sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA"
test "$(sudo sha256sum "$permission_verifier" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
test "$(sudo sha256sum "$service_env_verifier" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA"
test "$(sudo stat -c '%U:%G:%a' "$REMOTE_RUNTIME_BIN")" = "root:root:755"
test "$(sudo stat -c '%U:%G:%a' "$wrapper")" = "root:root:755"
test "$(sudo stat -c '%U:%G:%a' "$tool")" = "root:root:755"
test "$(sudo stat -c '%U:%G:%a' "$permission_verifier")" = "root:root:755"
test "$(sudo stat -c '%U:%G:%a' "$service_env_verifier")" = "root:root:755"
test "$(sudo stat -c '%U:%G:%a' "$revision")" = "root:root:644"
test "$(sudo stat -c '%U:%G:%a' "$dropin")" = "root:root:644"
test "$(sudo cat "$revision")" = "$SOURCE_REVISION"
for protected_dir in \
/usr \
/usr/local \
/usr/local/libexec \
/usr/local/libexec/livingip \
"$REMOTE_RUNTIME_BIN" \
"$REMOTE_DROPIN_DIR"; do
sudo test -d "$protected_dir"
sudo test ! -L "$protected_dir"
test "$(sudo stat -c '%U:%G' "$protected_dir")" = "root:root"
sudo -n -u teleo test ! -w "$protected_dir"
done
for protected_file in \
"$wrapper" \
"$tool" \
"$permission_verifier" \
"$service_env_verifier" \
"$revision" \
"$dropin"; do
sudo test -f "$protected_file"
sudo test ! -L "$protected_file"
sudo -n -u teleo test ! -w "$protected_file"
done
for executable_dir in \
/usr/local/sbin \
/usr/local/bin \
/usr/sbin \
/usr/bin \
/sbin \
/bin; do
sudo test -d "$executable_dir"
test "$(sudo stat -Lc '%U:%G' "$executable_dir")" = "root:root"
sudo -n -u teleo test ! -w "$executable_dir"
done
for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3; do
sudo test -x "$protected_file"
test "$(sudo stat -Lc '%U:%G' "$protected_file")" = "root:root"
sudo -n -u teleo test ! -w "$protected_file"
done
sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin"
sudo grep -E '^Environment=TELEO_(KB_MODE|GCP_PROJECT|GCP_METADATA_ONLY|CLOUDSQL_|CANONICAL_CLOUDSQL_DB)' "$dropin"
sudo grep -Fx "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$dropin"
sudo grep -Fx "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$dropin"
sudo grep -Fx "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$dropin"
sudo grep -Fx 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$dropin"
}
assert_deployed_files() {
local wrapper tool dropin
wrapper="$REMOTE_PROFILE_BIN/teleo-kb"
tool="$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py"
dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME"
test "$(sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA"
test "$(sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA"
test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA"
sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin"
sudo grep -E '^Environment=(CLOUDSDK_CONFIG|TELEO_(KB_MODE|GCP_PROJECT|CLOUDSQL_|CANONICAL_CLOUDSQL_DB))' "$dropin"
local main_pid namespace_state source_state mount_state
local hermes_mount_state state_mount_state workspace_mount_state capability_set
assert_installed_files
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
source_state="$(stat -Lc '%d:%i' "$REMOTE_RUNTIME_BIN")"
namespace_state="$(sudo nsenter -t "$main_pid" -m -- stat -Lc '%d:%i' "$SERVICE_PROFILE_BIN")"
if [ "$namespace_state" != "$source_state" ]; then
echo "Service namespace does not bind the reviewed runtime directory." >&2
return 1
fi
mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -T "$SERVICE_PROFILE_BIN" -n -o OPTIONS)"
case ",$mount_state," in
*,ro,*) ;;
*)
echo "Service runtime bind is not read-only." >&2
return 1
;;
esac
hermes_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_HERMES_ROOT" -n -o OPTIONS)"
state_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/state" -n -o OPTIONS)"
workspace_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/workspace" -n -o OPTIONS)"
case ",$hermes_mount_state," in
*,ro,*) ;;
*)
echo "Service .hermes ancestor mount is not read-only." >&2
return 1
;;
esac
case ",$state_mount_state," in
*,rw,*) ;;
*)
echo "Service state mount is not writable." >&2
return 1
;;
esac
case ",$workspace_mount_state," in
*,rw,*) ;;
*)
echo "Service workspace mount is not writable." >&2
return 1
;;
esac
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w /home
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT"
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT/profiles"
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_ROOT"
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_BIN"
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/state"
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/workspace"
capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)"
if printf '%s\n' "$capability_set" | grep -Eqi '(^|[[:space:]])cap_sys_admin([[:space:]]|$)'; then
echo "Service capability bounding set still contains CAP_SYS_ADMIN." >&2
return 1
fi
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb" | awk '{print $1}')" = "$WRAPPER_SHA"
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql_memory_tool.py" | awk '{print $1}')" = "$TOOL_SHA"
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_runtime_permissions.py" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_service_environment.py" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
}
REMOTE
)
printf -v verify_context \
'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\n' \
"$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \
"$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \
"$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA"
'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \
"$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \
"$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \
"$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" "$SOURCE_REVISION"
verify_body=$(cat <<'REMOTE'
set -euo pipefail
verification_tmp="$(mktemp -d /tmp/teleo-gcp-runtime-verify.XXXXXX)"
sudo chown teleo:teleo "$verification_tmp"
sudo chmod 0700 "$verification_tmp"
cleanup_verification() { sudo rm -rf -- "$verification_tmp"; }
trap cleanup_verification EXIT
assert_deployed_files
assert_service_running
assert_runtime_environment
run_scoped_status verify "$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py" "$verification_tmp/gcloud"
assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"
run_service_wrapper_status verify
assert_administrator_secret_denied
run_permission_verifier verify "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" "$verification_tmp/permission-receipt.json"
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
REMOTE
)
verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}"
if $DRY_RUN; then
echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE restart=$RESTART"
echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA dropin_sha256=$DROPIN_SHA"
echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE preflight_only=$PREFLIGHT_ONLY restart=$RESTART"
echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA dropin_sha256=$DROPIN_SHA"
exit 0
fi
@ -236,39 +548,88 @@ if $VERIFY_ONLY; then
exit 0
fi
if ! $RESTART; then
echo "Refusing to install runtime files without --restart; use --dry-run or --verify-only for read-only work." >&2
if ! $RESTART && ! $PREFLIGHT_ONLY; then
echo "Refusing to install runtime files without --restart; use --dry-run, --preflight-only, or --verify-only for read-only work." >&2
exit 64
fi
REMOTE_TMP="/tmp/teleo-gcp-leoclean-runtime-$(date -u +%Y%m%dT%H%M%SZ)-$$"
printf -v sync_context \
'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\nREMOTE_TMP=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\n' \
"$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \
"$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \
"$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA" "$REMOTE_TMP" \
"$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" "$SOURCE_REVISION"
'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \
"$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \
"$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \
"$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" \
"$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY"
sync_body=$(cat <<'REMOTE'
set -euo pipefail
remote_tmp="$REMOTE_TMP"
profile_bin="$REMOTE_PROFILE_BIN"
remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)"
runtime_bin="$REMOTE_RUNTIME_BIN"
runtime_parent="$(dirname "$runtime_bin")"
runtime_grandparent="$(dirname "$runtime_parent")"
runtime_anchor="$(dirname "$runtime_grandparent")"
dropin_dir="$REMOTE_DROPIN_DIR"
dropin_parent="$(dirname "$dropin_dir")"
payload_dir="$remote_tmp/payload"
backup_dir="$remote_tmp/backup"
work_dir="$remote_tmp/work"
wrapper_stage="$runtime_bin/.teleo-kb.livingip-new.$$"
tool_stage="$runtime_bin/.cloudsql-memory-tool.livingip-new.$$"
permission_verifier_stage="$runtime_bin/.runtime-permission-verifier.livingip-new.$$"
service_env_verifier_stage="$runtime_bin/.service-environment-verifier.livingip-new.$$"
revision_stage="$runtime_bin/.livingip-runtime-revision.livingip-new.$$"
dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$"
mutation_started=false
assert_existing_root_directory() {
local target="$1"
if sudo test -e "$target" || sudo test -L "$target"; then
sudo test -d "$target"
sudo test ! -L "$target"
test "$(sudo stat -c '%U:%G' "$target")" = "root:root"
sudo -n -u teleo test ! -w "$target"
fi
}
assert_existing_root_file() {
local target="$1"
if sudo test -e "$target" || sudo test -L "$target"; then
sudo test -f "$target"
sudo test ! -L "$target"
test "$(sudo stat -c '%U:%G' "$target")" = "root:root"
sudo -n -u teleo test ! -w "$target"
fi
}
backup_path() {
local target="$1" name="$2"
if sudo test -e "$target"; then
sudo cp -a -- "$target" "$backup_dir/$name"
touch "$backup_dir/$name.present"
sudo touch "$backup_dir/$name.present"
fi
}
backup_directory_state() {
local target="$1" name="$2"
if sudo test -d "$target"; then
sudo stat -c '%u %g %a' "$target" | sudo tee "$backup_dir/$name.metadata" >/dev/null
sudo touch "$backup_dir/$name.present"
fi
}
restore_directory_state() {
local target="$1" name="$2" uid gid mode
if sudo test -f "$backup_dir/$name.present"; then
read -r uid gid mode <<<"$(sudo cat "$backup_dir/$name.metadata")"
sudo chown "$uid:$gid" "$target"
sudo chmod "$mode" "$target"
elif sudo test -d "$target"; then
sudo rmdir -- "$target"
fi
}
restore_path() {
local target="$1" name="$2" restore_rc=0 restore_tmp
if [ -f "$backup_dir/$name.present" ]; then
if sudo test -f "$backup_dir/$name.present"; then
restore_tmp="${target}.livingip-rollback.$$"
sudo rm -f -- "$restore_tmp" || restore_rc=1
if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then
@ -283,15 +644,65 @@ restore_path() {
return "$restore_rc"
}
validate_restored_path() {
local target="$1" name="$2"
if sudo test -f "$backup_dir/$name.present"; then
sudo test -f "$target"
sudo test ! -L "$target"
sudo cmp -s -- "$backup_dir/$name" "$target"
test "$(sudo stat -c '%u:%g:%a' "$target")" = \
"$(sudo stat -c '%u:%g:%a' "$backup_dir/$name")"
else
sudo test ! -e "$target"
sudo test ! -L "$target"
fi
}
validate_restored_directory() {
local target="$1" name="$2" expected_state
if sudo test -f "$backup_dir/$name.present"; then
expected_state="$(sudo cat "$backup_dir/$name.metadata" | tr ' ' ':')"
sudo test -d "$target"
sudo test ! -L "$target"
test "$(sudo stat -c '%u:%g:%a' "$target")" = "$expected_state"
else
sudo test ! -e "$target"
sudo test ! -L "$target"
fi
}
rollback_runtime() {
local rollback_rc=0
echo "Deployment failed; restoring the complete previous runtime set." >&2
sudo systemctl stop "$SERVICE" || rollback_rc=1
restore_path "$profile_bin/teleo-kb" wrapper || rollback_rc=1
restore_path "$profile_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
restore_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1
restore_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
restore_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1
restore_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1
restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1
restore_path "$profile_bin/.livingip-runtime-revision" revision || rollback_rc=1
sudo systemctl daemon-reload || rollback_rc=1
restore_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1
restore_directory_state "$runtime_bin" runtime_bin || rollback_rc=1
restore_directory_state "$runtime_parent" runtime_parent || rollback_rc=1
restore_directory_state "$runtime_grandparent" runtime_grandparent || rollback_rc=1
restore_directory_state "$dropin_dir" dropin_dir || rollback_rc=1
if [ "$rollback_rc" -eq 0 ]; then
validate_restored_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1
validate_restored_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
validate_restored_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1
validate_restored_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1
validate_restored_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1
validate_restored_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1
validate_restored_directory "$runtime_grandparent" runtime_grandparent || rollback_rc=1
validate_restored_directory "$runtime_parent" runtime_parent || rollback_rc=1
validate_restored_directory "$runtime_bin" runtime_bin || rollback_rc=1
validate_restored_directory "$dropin_dir" dropin_dir || rollback_rc=1
fi
if [ "$rollback_rc" -eq 0 ]; then
sudo systemctl daemon-reload || rollback_rc=1
fi
if [ "$rollback_rc" -eq 0 ]; then
assert_unit_configuration false || rollback_rc=1
fi
if [ "$rollback_rc" -eq 0 ]; then
sudo systemctl start "$SERVICE" || rollback_rc=1
assert_service_running || rollback_rc=1
@ -312,7 +723,13 @@ on_exit() {
rc=1
fi
fi
sudo rm -f -- "$dropin_stage" || true
sudo rm -f -- \
"$wrapper_stage" \
"$tool_stage" \
"$permission_verifier_stage" \
"$service_env_verifier_stage" \
"$revision_stage" \
"$dropin_stage" || true
sudo rm -rf -- "$remote_tmp" || true
exit "$rc"
}
@ -321,36 +738,117 @@ trap 'on_exit $?' EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
sudo rm -rf -- "$remote_tmp"
mkdir -p "$backup_dir"
tar -C "$remote_tmp" -xf -
sudo chmod 0711 "$remote_tmp"
sudo install -d -o root -g root -m 0755 "$payload_dir"
sudo install -d -o root -g root -m 0700 "$backup_dir"
sudo install -d -o teleo -g teleo -m 0700 "$work_dir"
sudo tar --no-same-owner --no-same-permissions --no-xattrs --no-acls --no-selinux -C "$payload_dir" -xf -
if sudo find "$payload_dir" -type l -print -quit | grep -q .; then
echo "Staged runtime payload contains a symlink." >&2
exit 65
fi
if sudo find "$payload_dir" ! -type d ! -type f -print -quit | grep -q .; then
echo "Staged runtime payload contains a non-regular entry." >&2
exit 65
fi
sudo chown -R root:root "$payload_dir"
sudo find "$payload_dir" -type d -exec chmod 0555 {} +
sudo find "$payload_dir" -type f -exec chmod 0444 {} +
sudo chmod 0555 \
"$payload_dir/$WRAPPER_REL" \
"$payload_dir/$TOOL_REL" \
"$payload_dir/$PERMISSION_VERIFIER_REL" \
"$payload_dir/$SERVICE_ENV_VERIFIER_REL"
test "$(sudo sha256sum "$payload_dir/$WRAPPER_REL" | awk '{print $1}')" = "$WRAPPER_SHA"
test "$(sudo sha256sum "$payload_dir/$TOOL_REL" | awk '{print $1}')" = "$TOOL_SHA"
test "$(sudo sha256sum "$payload_dir/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
test "$(sudo sha256sum "$payload_dir/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
test "$(sudo sha256sum "$payload_dir/$DROPIN_REL" | awk '{print $1}')" = "$DROPIN_SHA"
test "$(sudo stat -c '%U:%G:%a' "$remote_tmp")" = "root:root:711"
test "$(sudo stat -c '%U:%G:%a' "$payload_dir")" = "root:root:555"
test "$(sudo stat -c '%U:%G:%a' "$backup_dir")" = "root:root:700"
test "$(sudo stat -c '%U:%G:%a' "$work_dir")" = "teleo:teleo:700"
sudo -n -u teleo test ! -w "$remote_tmp"
sudo -n -u teleo test ! -w "$payload_dir"
sudo -n -u teleo test ! -w "$backup_dir"
sudo test -d "$runtime_anchor"
sudo test -d "$dropin_parent"
sudo test ! -L "$runtime_anchor"
sudo test ! -L "$dropin_parent"
test "$(sudo stat -c '%U:%G' "$runtime_anchor")" = "root:root"
test "$(sudo stat -c '%U:%G' "$dropin_parent")" = "root:root"
sudo -n -u teleo test ! -w "$runtime_anchor"
sudo -n -u teleo test ! -w "$dropin_parent"
for existing_directory in \
"$runtime_grandparent" \
"$runtime_parent" \
"$runtime_bin" \
"$dropin_dir"; do
assert_existing_root_directory "$existing_directory"
done
for existing_file in \
"$runtime_bin/teleo-kb" \
"$runtime_bin/cloudsql_memory_tool.py" \
"$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" \
"$runtime_bin/verify_gcp_leoclean_service_environment.py" \
"$runtime_bin/.livingip-runtime-revision" \
"$dropin_dir/$DROPIN_NAME"; do
assert_existing_root_file "$existing_file"
done
assert_service_running
run_scoped_status pre "$remote_tmp/$TOOL_REL" "$remote_tmp/gcloud-pre"
preflight_service_before="$(service_fingerprint)"
assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false
run_scoped_status pre "$payload_dir/$TOOL_REL"
assert_administrator_secret_denied
run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json"
assert_service_fingerprint "$preflight_service_before"
if [ "$PREFLIGHT_ONLY" = true ]; then
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
exit 0
fi
backup_path "$profile_bin/teleo-kb" wrapper
backup_path "$profile_bin/cloudsql_memory_tool.py" tool
backup_path "$runtime_bin/teleo-kb" wrapper
backup_path "$runtime_bin/cloudsql_memory_tool.py" tool
backup_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier
backup_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier
backup_path "$dropin_dir/$DROPIN_NAME" dropin
backup_path "$profile_bin/.livingip-runtime-revision" revision
backup_path "$runtime_bin/.livingip-runtime-revision" revision
backup_directory_state "$runtime_parent" runtime_parent
backup_directory_state "$runtime_grandparent" runtime_grandparent
backup_directory_state "$runtime_bin" runtime_bin
backup_directory_state "$dropin_dir" dropin_dir
mutation_started=true
sudo systemctl stop "$SERVICE"
sudo install -d -o teleo -g teleo -m 0755 "$profile_bin"
sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$WRAPPER_REL" "$profile_bin/teleo-kb"
sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$TOOL_REL" "$profile_bin/cloudsql_memory_tool.py"
sudo install -d -o root -g root -m 0755 "$runtime_parent"
sudo install -d -o root -g root -m 0755 "$runtime_bin"
sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage"
sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage"
sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" "$permission_verifier_stage"
sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" "$service_env_verifier_stage"
sudo mv -f -- "$wrapper_stage" "$runtime_bin/teleo-kb"
sudo mv -f -- "$tool_stage" "$runtime_bin/cloudsql_memory_tool.py"
sudo mv -f -- "$permission_verifier_stage" "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py"
sudo mv -f -- "$service_env_verifier_stage" "$runtime_bin/verify_gcp_leoclean_service_environment.py"
sudo install -d -o root -g root -m 0755 "$dropin_dir"
sudo install -o root -g root -m 0644 "$remote_tmp/$DROPIN_REL" "$dropin_stage"
sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL" "$dropin_stage"
sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME"
printf '%s\n' "$SOURCE_REVISION" | sudo tee "$profile_bin/.livingip-runtime-revision" >/dev/null
sudo chown teleo:teleo "$profile_bin/.livingip-runtime-revision"
sudo chmod 0644 "$profile_bin/.livingip-runtime-revision"
printf '%s\n' "$SOURCE_REVISION" | sudo tee "$revision_stage" >/dev/null
sudo chown root:root "$revision_stage"
sudo chmod 0644 "$revision_stage"
sudo mv -f -- "$revision_stage" "$runtime_bin/.livingip-runtime-revision"
assert_installed_files
sudo systemctl daemon-reload
assert_unit_configuration
sudo systemctl start "$SERVICE"
assert_deployed_files
assert_service_running
assert_runtime_environment
run_scoped_status post "$profile_bin/cloudsql_memory_tool.py" "$remote_tmp/gcloud-post"
assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py"
run_service_wrapper_status post
assert_administrator_secret_denied
run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" "$work_dir/permission-post/receipt.json"
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
mutation_started=false
REMOTE
@ -358,4 +856,8 @@ REMOTE
sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}"
COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \
"$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command"
"$WRAPPER_REL" \
"$TOOL_REL" \
"$PERMISSION_VERIFIER_REL" \
"$SERVICE_ENV_VERIFIER_REL" \
"$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command"

View file

@ -2,8 +2,9 @@
## Scope
This runbook reconciles PR `#145` with the non-production GCP parallel Leo
service and removes its dependency on the PostgreSQL administrator credential.
This runbook reconciles PR `#148`, the least-privilege follow-up to PR `#145`,
with the non-production GCP parallel Leo service and removes its dependency on
the PostgreSQL administrator credential.
It does not promote GCP, change the Telegram destination, copy newer VPS data,
or make `teleo_canonical` production-authoritative.
@ -31,7 +32,8 @@ IAM or networking.
commit.
2. `TELEO_KB_MODE=cloudsql`; missing tools or credentials fail closed.
3. Canonical zero-hit searches do not consult `teleo_restore` unless an
operator explicitly opts in.
operator explicitly opts in. The service is pinned to database
`teleo_canonical` with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`.
4. Leo can read the named canonical tables and stage only a
`pending_review` proposal through `kb_stage.stage_leoclean_proposal(...)`.
5. Leo cannot directly insert, update, or delete `public.*` or
@ -55,7 +57,7 @@ required runtime secret has an equivalent secret-level binding.
### 2. Merge reviewed repository code
Run CI on the completed PR `#145` repair before deployment. The deployment
Run CI on the completed PR `#148` repair before deployment. The deployment
source must be the resulting merged commit, not a working tree or unmerged
branch.
@ -73,6 +75,9 @@ private VM path. Supply the new runtime password through
`TELEO_LEOCLEAN_DB_PASSWORD`; the SQL file does not contain it. The migration:
- creates or rotates `leoclean_kb_runtime`;
- keeps `leoclean_kb_runtime` `NOLOGIN` while both the `teleo_canonical` grant
phase and the legacy `teleo_kb` denial-verification phase run, and enables
`LOGIN` only in the final transaction after both phases pass;
- creates a dedicated `NOLOGIN` function owner with no role memberships;
- removes stale table, column, sequence, function, and role-membership grants;
- grants exact canonical reads;
@ -94,26 +99,116 @@ by a `pg_catalog, pg_temp` search path, schema-qualified relations, a fixed
The administrator password is used only for this bounded bootstrap. Retain no
password output.
`teleo_kb` can still be connectable through its existing `PUBLIC CONNECT`
grant. That database-wide default is not treated as runtime authority. The
migration denies the scoped role access to the `teleo_restore` schema, the
runtime client is pinned to `teleo_canonical`, and audit fallback remains
disabled with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`.
### 5. Preflight and deploy the merged runtime
From the exact merged checkout, run:
```bash
deploy/sync-gcp-leoclean-runtime.sh --dry-run
deploy/sync-gcp-leoclean-runtime.sh --preflight-only
deploy/sync-gcp-leoclean-runtime.sh --restart
```
The deploy script runs a scoped, redacted database status preflight before it
installs files or restarts the service. It refuses dirty or unmerged runtime
files and refuses a live install without `--restart`. After preflight it stops
the parallel service, atomically replaces the existing Cloud SQL systemd
drop-in, installs the reviewed wrapper and helper, records the Git revision,
starts the service, and verifies all three file hashes. A failure after mutation
restores the prior wrapper, helper, drop-in, and revision as one set before the
old service is restarted.
`--preflight-only` extracts the candidate helper and permission verifier into a
root-created, root-owned, non-writable VM payload directory, verifies the actual
service `MainPID` environment, runs the candidate private Cloud SQL status path,
and runs the sanitized positive-and-negative permission verifier. A separate
`teleo`-owned directory holds only sanitized receipts; it never feeds an install
or rollback. The preflight does not stop or restart the service and does not
install or replace runtime files. It
captures `ActiveState`, `SubState`, `MainPID`, and `NRestarts` before and after
the checks and fails if any value changes. The preflight must pass before
`--restart` is attempted.
The deploy script refuses dirty, symlinked, or unmerged runtime files and
refuses a live install without `--restart`. Its payload and rollback backup are
root-owned and non-writable by `teleo`; existing runtime intermediates and
targets must be real root-owned, non-writable paths. After preflight it stops
the parallel service, atomically replaces the existing Cloud SQL systemd drop-in, installs the
reviewed wrapper, helper, and permission verifier into the root-controlled
`/usr/local/libexec/livingip/leoclean-kb` directory, records the Git revision,
and verifies every installed hash, owner, mode, path type, and revision before
systemd reloads the unit or starts the service. The wrapper uses fixed
`/bin/bash` and `/usr/bin/python3` interpreters, and every directory on the
service `PATH` must be root-owned and non-writable by `teleo`. Every ancestor of that
directory, the directory itself, the wrapper, helper, verifier, and revision
marker must be non-writable by `teleo`. The systemd service receives that
directory as a read-only bind at its existing
`/home/teleo/.hermes/profiles/leoclean/bin` path; verification compares the
source and service-namespace inode and hashes and proves the mount is read-only.
To prevent the `teleo` service user from renaming a writable ancestor and
recreating that absolute path, the service namespace mounts
`/home/teleo/.hermes` read-only, allows writes back only to the profile's
`state` and `workspace` subdirectories, and removes `CAP_SYS_ADMIN`. Live
verification requires `.hermes` and `bin` to be read-only, both allowed
subdirectories to be writable, `/home` to be non-writable by `teleo`, and the
effective capability set to exclude `CAP_SYS_ADMIN`.
Preflight, post-restart, and `--verify-only` checks inspect the environment of the actual
systemd `MainPID` through `/proc/<MainPID>/environ`, rather than trusting the
configured unit environment alone. A standalone fail-closed verifier requires
the exact reviewed private target and runtime role, requires
`TELEO_GCP_METADATA_ONLY=1`, and rejects all PostgreSQL, Cloud SDK, proxy, TLS
trust, broader Google credential, shell-startup, dynamic-loader, Python-loader,
and administrator-secret overrides without printing their values. Effective
`EnvironmentFile` use and drop-ins ordered after the reviewed configuration are
rejected before restart. This blocks
pre-wrapper injection through fields such as `BASH_ENV`, any `LD_*`,
`PYTHONPATH`, `PYTHONHTTPSVERIFY`, or `SSLKEYLOGFILE`. The checks also require
the reviewed drop-in to appear in `DropInPaths`. The wrapper status probe then
imports the validated environment from the actual `MainPID` with `nsenter
--env`, enters its mount namespace, drops to `teleo`, and invokes the bound
wrapper without reinjecting any database identity setting.
The service runtime bypasses Cloud SDK configuration and credential caches
entirely. The helper obtains the attached service-account email and a bounded
access token from the fixed GCE metadata endpoint, requires the exact expected
service account, and accesses only the named scoped Secret Manager version over
the fixed HTTPS API. Its URL opener disables proxies. The scoped status read
and administrator-secret IAM denial are rerun after restart. The denial probe
imports the actual systemd `MainPID` environment and mount namespace, drops to
the service's `teleo` identity, and must observe the exact
`secretmanager.versions.access` permission denial. No token or
secret value is retained. The status read executes the bound `teleo-kb` wrapper
from inside the service mount namespace, proving the wrapper and its adjacent
helper rather than bypassing them with a direct helper invocation. A failure
after mutation restores the prior runtime/drop-in directory existence and
metadata, wrapper, helper, both verifiers, drop-in, and revision as one set,
validates it against the root-only backup, and only then reloads systemd and
restarts the old service.
### 6. Verify positive and negative behavior
`--preflight-only`, `--restart`, and `--verify-only` run the verifier as the
`teleo` Unix user. To rerun the installed verifier directly on the VM and
retain its sanitized JSON receipt, use:
```bash
receipt_dir="$(mktemp -d /tmp/leoclean-permission-receipt.XXXXXX)"
sudo chown teleo:teleo "$receipt_dir"
sudo chmod 0700 "$receipt_dir"
run_id="$(date -u +%Y%m%d%H%M%S)-manual"
sudo -n -u teleo env -i HOME=/home/teleo \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
python3 /usr/local/libexec/livingip/leoclean-kb/verify_gcp_leoclean_runtime_permissions.py \
--run-id "$run_id" \
--output "$receipt_dir/permission-receipt.json"
sudo stat -c '%U:%G:%a' "$receipt_dir/permission-receipt.json"
```
The command must exit zero, print the same canonical JSON that it writes, and
leave the receipt as `teleo:teleo:600`. A passing receipt must report
`status=pass`, `mode=live_private_gcp_staging`, and both `required_tier` and
`current_tier` as `T3_live_readonly`. Those values are a receipt contract, not
a claim that this runbook has already been exercised successfully on the live
staging VM.
Retain a redacted receipt containing:
- merged Git commit and deployed file hashes;
@ -122,13 +217,28 @@ Retain a redacted receipt containing:
configuration;
- `current_database()` and `current_user` showing
`teleo_canonical|leoclean_kb_runtime`;
- private server address `10.61.0.3`, port `5432`, and an active SSL session;
- exact safe runtime-role attributes and zero membership edges in
`pg_auth_members`;
- zero effective database/schema `CREATE`, persistent ownership, table/column
DML, sequence privileges, SELECT outside the exact table allowlist, or
executable unexpected `SECURITY DEFINER` functions, plus the exact staging
function owner/search-path/ACL contract;
- a real canonical status/search receipt;
- a transaction-rolled-back call to `stage_leoclean_proposal(...)` that returns
`pending_review`;
- denied direct insert into `kb_stage.kb_proposals`;
- denied canonical `public.*` write;
- denied reviewer/apply security-definer function;
- denied forged proposer identity and denied `SET ROLE` escalation;
`pending_review` for canonical proposer `leo`, with zero matching canary rows
both before and after the transaction;
- denied direct insert, update, and delete on `kb_stage.kb_proposals`;
- denied insert, update, and delete on canonical `public.claims`;
- denied reviewer/apply security-definer functions;
- exact function-catalog posture: only the five-argument staging function is
executable, the forged six-argument overload is absent, and each expected
reviewer/apply function exists but is not executable by the runtime role;
- unavailable forged-proposer overload and denied `SET ROLE` escalation into
the staging owner, reviewer, apply, or administrator roles;
- denied access to `teleo_restore` even when `teleo_kb` remains connectable;
- readable scoped runtime secret and an IAM permission denial for the
administrator secret, with neither secret value retained;
- zero Telegram messages and zero committed canary rows.
### 7. Remove administrator-secret access

View file

@ -1,20 +1,24 @@
#!/usr/bin/env python3
"""Cloud SQL memory and KB-proposal bridge for the GCP leoclean profile.
Canonical claim/strategy application remains review-gated. Read commands query
the canonical Teleo KB first, then fall back to the restored `teleo_restore`
shadow schema in Cloud SQL. Proposal commands write staged review records into
`kb_stage.kb_proposals`; they do not apply canonical truth directly.
Canonical claim/strategy application remains review-gated. The production-like
Leo runtime is pinned to the private canonical Cloud SQL database and never
falls back to the restored `teleo_restore` shadow schema. Proposal commands
write staged review records into `kb_stage.kb_proposals`; they do not apply
canonical truth directly.
"""
from __future__ import annotations
import argparse
import base64
import binascii
import hashlib
import json
import os
import re
import subprocess
import urllib.request
from typing import Any
STOPWORDS = {
@ -76,6 +80,38 @@ DEFAULT_CLAIM_BASE_URL = "https://leo.livingip.xyz"
RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1"
RUNTIME_DB_USER = "leoclean_kb_runtime"
RUNTIME_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password"
RUNTIME_GCP_PROJECT = "teleo-501523"
RUNTIME_CLOUDSQL_HOST = "10.61.0.3"
RUNTIME_CLOUDSQL_PORT = "5432"
RUNTIME_CLOUDSQL_DB = "teleo_canonical"
RUNTIME_SERVICE_ACCOUNT = "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com"
GCE_METADATA_EMAIL_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email"
GCE_METADATA_TOKEN_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token"
RUNTIME_SECRET_ACCESS_URL = (
"https://secretmanager.googleapis.com/v1/projects/teleo-501523/secrets/"
"gcp-teleo-pgvector-standby-leoclean-kb-runtime-password/versions/latest:access"
)
METADATA_HEADERS = {"Metadata-Flavor": "Google"}
HTTP_TIMEOUT_SECONDS = 3.0
MAX_METADATA_EMAIL_BYTES = 256
MAX_METADATA_TOKEN_BYTES = 4096
MAX_SECRET_RESPONSE_BYTES = 8192
MAX_RUNTIME_SECRET_BYTES = 4096
RUNTIME_TLS_TRUST_ENVIRONMENT = frozenset(
{
"CURL_CA_BUNDLE",
"PYTHONHTTPSVERIFY",
"REQUESTS_CA_BUNDLE",
"SSL_CERT_DIR",
"SSL_CERT_FILE",
"SSLKEYLOGFILE",
}
)
RUNTIME_PSQL_ENV_BASE = {
"PATH": "/usr/bin:/bin",
"PGSSLMODE": "require",
"PGCONNECT_TIMEOUT": "8",
}
CLONE_READ_ONLY_PGOPTIONS = "-c default_transaction_read_only=on"
RECEIPTED_READ_COMMANDS = frozenset(
{
@ -93,6 +129,18 @@ RECEIPTED_READ_COMMANDS = frozenset(
)
class _RejectRedirectHandler(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *_args: Any, **_kwargs: Any) -> None:
return None
RUNTIME_PROXY_HANDLER = urllib.request.ProxyHandler({})
RUNTIME_HTTP_OPENER = urllib.request.build_opener(
RUNTIME_PROXY_HANDLER,
_RejectRedirectHandler(),
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default=os.environ.get("TELEO_CLOUDSQL_HOST", "10.61.0.3"))
@ -246,8 +294,47 @@ def validate_runtime_connection(args: argparse.Namespace) -> None:
raise SystemExit(f"Leo runtime requires database user {RUNTIME_DB_USER}.")
if password_secret != RUNTIME_PASSWORD_SECRET:
raise SystemExit(f"Leo runtime requires scoped password secret {RUNTIME_PASSWORD_SECRET}.")
if "PGPASSWORD" in os.environ:
raise SystemExit("Refusing inherited PGPASSWORD in Leo runtime mode.")
expected_fields = {
"project": RUNTIME_GCP_PROJECT,
"host": RUNTIME_CLOUDSQL_HOST,
"port": RUNTIME_CLOUDSQL_PORT,
"db": RUNTIME_CLOUDSQL_DB,
"canonical_db": RUNTIME_CLOUDSQL_DB,
}
for field, expected in expected_fields.items():
actual = str(getattr(args, field, "") or "")
if actual != expected:
raise SystemExit(f"Leo runtime requires {field}={expected}.")
if os.environ.get("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") != "0":
raise SystemExit("Leo runtime requires TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0.")
if os.environ.get("TELEO_GCP_METADATA_ONLY") != "1":
raise SystemExit("Leo runtime requires TELEO_GCP_METADATA_ONLY=1.")
forbidden_environment = any(
normalized_key.startswith("PG")
or normalized_key.startswith("CLOUDSDK_")
or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS"
or normalized_key.startswith("LD_")
or normalized_key.startswith("BASH_FUNC_")
or (normalized_key.startswith("PYTHON") and normalized_key != "PYTHONUNBUFFERED")
or normalized_key in {
"BASH_ENV",
"BASHOPTS",
"CDPATH",
"ENV",
"GLOBIGNORE",
"IFS",
"PS4",
"SHELLOPTS",
}
or normalized_key == "PROXY"
or normalized_key.endswith("_PROXY")
or normalized_key in RUNTIME_TLS_TRUST_ENVIRONMENT
for normalized_key in (key.upper() for key in os.environ)
)
if forbidden_environment:
raise SystemExit(
"Refusing inherited credential, proxy, TLS, or PostgreSQL environment in Leo runtime mode."
)
return
if mode != "clone-readonly":
@ -261,8 +348,8 @@ def validate_runtime_connection(args: argparse.Namespace) -> None:
raise SystemExit("Clone credential mode permits read commands only.")
if pgoptions != CLONE_READ_ONLY_PGOPTIONS:
raise SystemExit("Clone credential mode requires server-enforced default_transaction_read_only=on.")
if not user or (not password_secret and not os.environ.get("PGPASSWORD")):
raise SystemExit("Clone credential mode requires an explicit user and credential source.")
if not user or not os.environ.get("PGPASSWORD"):
raise SystemExit("Clone credential mode requires an explicit user and inherited test credential.")
def terms_for(query: str) -> list[str]:
@ -305,48 +392,138 @@ def sql_array(values: list[str], cast: str = "text") -> str:
return "array[" + ",".join(sql_literal(value) for value in values) + f"]::{cast}[]"
def _read_http_response(request: urllib.request.Request, *, max_bytes: int, purpose: str) -> bytes:
try:
with RUNTIME_HTTP_OPENER.open(request, timeout=HTTP_TIMEOUT_SECONDS) as response:
if response.getcode() != 200:
raise SystemExit(f"{purpose} request failed.")
body = response.read(max_bytes + 1)
except SystemExit:
raise
# This is a credential-bearing boundary. Never propagate provider, proxy,
# TLS, or test-double exception text because it can contain bearer data.
except Exception:
raise SystemExit(f"{purpose} request failed.") from None
if not isinstance(body, bytes) or len(body) > max_bytes:
raise SystemExit(f"{purpose} response was invalid.")
return body
def _strict_json_object(body: bytes, *, purpose: str) -> dict[str, Any]:
def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
value: dict[str, Any] = {}
for key, item in pairs:
if key in value:
raise ValueError("duplicate JSON key")
value[key] = item
return value
try:
decoded = body.decode("utf-8", errors="strict")
value = json.loads(
decoded,
object_pairs_hook=reject_duplicate_keys,
parse_constant=lambda _constant: (_ for _ in ()).throw(ValueError("non-finite JSON value")),
)
except (UnicodeDecodeError, ValueError, TypeError):
raise SystemExit(f"{purpose} response was invalid.") from None
if not isinstance(value, dict):
raise SystemExit(f"{purpose} response was invalid.")
return value
def _metadata_request(url: str, *, max_bytes: int, purpose: str) -> bytes:
request = urllib.request.Request(url, headers=METADATA_HEADERS, method="GET")
return _read_http_response(request, max_bytes=max_bytes, purpose=purpose)
def runtime_password_from_metadata() -> str:
email_body = _metadata_request(
GCE_METADATA_EMAIL_URL,
max_bytes=MAX_METADATA_EMAIL_BYTES,
purpose="Runtime metadata identity",
)
try:
service_account = email_body.decode("ascii", errors="strict")
except UnicodeDecodeError:
raise SystemExit("Runtime metadata identity response was invalid.") from None
if service_account != RUNTIME_SERVICE_ACCOUNT:
raise SystemExit("Runtime metadata identity did not match the required service account.")
token_value = _strict_json_object(
_metadata_request(
GCE_METADATA_TOKEN_URL,
max_bytes=MAX_METADATA_TOKEN_BYTES,
purpose="Runtime metadata token",
),
purpose="Runtime metadata token",
)
access_token = token_value.get("access_token")
token_type = token_value.get("token_type")
expires_in = token_value.get("expires_in")
if (
token_type != "Bearer"
or not isinstance(access_token, str)
or not re.fullmatch(r"[!-~]{1,4096}", access_token)
or not isinstance(expires_in, int)
or isinstance(expires_in, bool)
or expires_in <= 0
):
raise SystemExit("Runtime metadata token response was invalid.")
secret_request = urllib.request.Request(
RUNTIME_SECRET_ACCESS_URL,
headers={"Accept": "application/json", "Authorization": f"Bearer {access_token}"},
method="GET",
)
secret_value = _strict_json_object(
_read_http_response(
secret_request,
max_bytes=MAX_SECRET_RESPONSE_BYTES,
purpose="Scoped runtime secret access",
),
purpose="Scoped runtime secret access",
)
payload = secret_value.get("payload")
encoded_secret = payload.get("data") if isinstance(payload, dict) else None
if not isinstance(encoded_secret, str):
raise SystemExit("Scoped runtime secret response was invalid.")
try:
credential_bytes = base64.b64decode(encoded_secret, validate=True)
except (binascii.Error, ValueError):
raise SystemExit("Scoped runtime secret response was invalid.") from None
if not credential_bytes or len(credential_bytes) > MAX_RUNTIME_SECRET_BYTES:
raise SystemExit("Scoped runtime secret response was invalid.")
if any(byte in credential_bytes for byte in (0, 10, 13)):
raise SystemExit("Scoped runtime secret response was invalid.")
try:
return credential_bytes.decode("utf-8", errors="strict")
except UnicodeDecodeError:
raise SystemExit("Scoped runtime secret response was invalid.") from None
def password(args: argparse.Namespace) -> str:
validate_runtime_connection(args)
if args.credential_mode == "clone-readonly" and os.environ.get("PGPASSWORD"):
if args.credential_mode == "runtime":
return runtime_password_from_metadata()
if os.environ.get("PGPASSWORD"):
return os.environ["PGPASSWORD"]
if not args.password_secret:
raise SystemExit("Scoped Cloud SQL password secret is not configured.")
result = subprocess.run(
[
"gcloud",
"secrets",
"versions",
"access",
"latest",
f"--secret={args.password_secret}",
f"--project={args.project}",
],
text=True,
capture_output=True,
check=False,
)
if result.returncode != 0:
raise SystemExit(f"Secret access failed: {result.stderr.strip()}")
credential = result.stdout.strip()
if not credential:
raise SystemExit("Scoped Cloud SQL password secret resolved to an empty value.")
return credential
raise SystemExit("Clone credential mode requires an inherited test credential.")
def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str:
env = os.environ.copy()
for key in tuple(env):
if key.startswith("PG"):
env.pop(key, None)
credential = password(args)
if args.credential_mode == "runtime":
env = dict(RUNTIME_PSQL_ENV_BASE)
else:
env = {key: value for key, value in os.environ.items() if not key.startswith("PG")}
env.update(
{
"PGHOST": str(args.host),
"PGPORT": str(args.port),
"PGDATABASE": str(db or args.db),
"PGUSER": str(args.user),
"PGSSLMODE": "require",
"PGCONNECT_TIMEOUT": "8",
"PGPASSWORD": password(args),
"PGPASSWORD": credential,
}
)
if args.credential_mode == "clone-readonly":
@ -360,9 +537,7 @@ def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str:
check=False,
)
if result.returncode != 0:
raise SystemExit(
f"Cloud SQL query failed ({result.returncode})\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
)
raise SystemExit(f"Cloud SQL query failed with exit status {result.returncode}.")
return result.stdout
@ -385,6 +560,10 @@ def database_read_marker(args: argparse.Namespace) -> dict[str, Any]:
select jsonb_build_object(
'database', current_database(),
'database_user', current_user,
'server_addr', inet_server_addr()::text,
'server_port', inet_server_port(),
'ssl', coalesce((select ssl from pg_catalog.pg_stat_ssl where pid = pg_backend_pid()), false),
'ssl_version', (select version from pg_catalog.pg_stat_ssl where pid = pg_backend_pid()),
'system_identifier', (select system_identifier::text from pg_control_system()),
'wal_lsn', pg_current_wal_lsn()::text
)::text;
@ -392,12 +571,34 @@ select jsonb_build_object(
rows = psql_json_lines(args, sql, db=args.canonical_db)
if len(rows) != 1:
raise SystemExit("Canonical database read marker returned an invalid row count.")
return rows[0]
marker = rows[0]
expected_database = str(args.canonical_db or "")
expected_user = str(args.user or "")
if marker.get("database") != expected_database or marker.get("database_user") != expected_user:
raise SystemExit("Canonical database read marker returned an unexpected database identity.")
if getattr(args, "credential_mode", "runtime") == "runtime":
if marker.get("server_addr") != RUNTIME_CLOUDSQL_HOST:
raise SystemExit("Canonical database read marker returned an unexpected server address.")
if str(marker.get("server_port")) != RUNTIME_CLOUDSQL_PORT:
raise SystemExit("Canonical database read marker returned an unexpected server port.")
if marker.get("ssl") is not True:
raise SystemExit("Canonical database read marker did not prove an encrypted connection.")
return marker
def _read_marker_stable(before: dict[str, Any], after: dict[str, Any]) -> bool:
return all(
before.get(key) == after.get(key) for key in ("database", "database_user", "system_identifier", "wal_lsn")
before.get(key) == after.get(key)
for key in (
"database",
"database_user",
"server_addr",
"server_port",
"ssl",
"ssl_version",
"system_identifier",
"wal_lsn",
)
)
@ -470,6 +671,10 @@ def build_retrieval_receipt(
"attempts": attempts,
"database": before.get("database"),
"database_user": before.get("database_user"),
"server_addr": before.get("server_addr"),
"server_port": before.get("server_port"),
"ssl": before.get("ssl"),
"ssl_version": before.get("ssl_version"),
"system_identifier": before.get("system_identifier"),
"wal_lsn_before": before.get("wal_lsn"),
"wal_lsn_after": after.get("wal_lsn"),

View file

@ -1,9 +1,15 @@
#!/usr/bin/env bash
#!/bin/bash
set -euo pipefail
PROFILE_DIR=${HERMES_HOME:-/home/teleo/.hermes/profiles/leoclean}
KB_TOOL="$PROFILE_DIR/bin/kb_tool.py"
CLOUDSQL_TOOL="$PROFILE_DIR/bin/cloudsql_memory_tool.py"
SCRIPT_PATH=${BASH_SOURCE[0]}
SCRIPT_DIR=${SCRIPT_PATH%/*}
if [ "$SCRIPT_DIR" = "$SCRIPT_PATH" ]; then
SCRIPT_DIR=.
fi
SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"
CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py"
MODE=${TELEO_KB_MODE:-auto}
COMMAND=${1:-}
@ -31,17 +37,22 @@ run_cloudsql() {
echo "teleo-kb: Cloud SQL tool is missing or not executable: $CLOUDSQL_TOOL" >&2
exit 69
fi
for dependency in python3 psql gcloud; do
if ! command -v "$dependency" >/dev/null 2>&1; then
echo "teleo-kb: required Cloud SQL dependency is unavailable: $dependency" >&2
exit 69
fi
done
if [ ! -x /usr/bin/python3 ]; then
echo "teleo-kb: required Cloud SQL dependency is unavailable: /usr/bin/python3" >&2
exit 69
fi
if ! command -v psql >/dev/null 2>&1; then
echo "teleo-kb: required Cloud SQL dependency is unavailable: psql" >&2
exit 69
fi
extra_args=()
while IFS= read -r arg; do
[ -n "$arg" ] && extra_args+=("$arg")
done < <(cloudsql_args)
exec python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}"
if [ "${#extra_args[@]}" -gt 0 ]; then
exec /usr/bin/python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}"
fi
exec /usr/bin/python3 "$CLOUDSQL_TOOL" "$@"
}
case "$MODE" in
@ -49,10 +60,10 @@ case "$MODE" in
run_cloudsql "$@"
;;
local)
exec python3 "$KB_TOOL" --local "$@"
exec /usr/bin/python3 "$KB_TOOL" --local "$@"
;;
remote)
exec python3 "$KB_TOOL" "$@"
exec /usr/bin/python3 "$KB_TOOL" "$@"
;;
auto)
# Canonical commands must fail closed on Cloud SQL errors. Falling through
@ -62,9 +73,9 @@ case "$MODE" in
run_cloudsql "$@"
fi
if command -v docker >/dev/null 2>&1; then
exec python3 "$KB_TOOL" --local "$@"
exec /usr/bin/python3 "$KB_TOOL" --local "$@"
fi
exec python3 "$KB_TOOL" "$@"
exec /usr/bin/python3 "$KB_TOOL" "$@"
;;
*)
echo "Unsupported TELEO_KB_MODE=$MODE; expected auto, cloudsql, local, or remote" >&2

View file

@ -1,5 +1,4 @@
\set ON_ERROR_STOP on
\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD
-- One-time GCP staging provisioning for Leo's Cloud SQL identity. The password
-- is supplied only through the process environment and is never stored here.
@ -8,7 +7,10 @@
-- through one narrowly-scoped SECURITY DEFINER function. It cannot inherit or
-- SET ROLE into another principal, forge the proposer identity, write directly
-- to canonical/staging tables, or inspect the legacy teleo_restore audit schema.
select 'create role leoclean_kb_runtime login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8'
-- Keep the runtime unable to authenticate until both database phases have
-- completed and verified. Any failure before the final transaction therefore
-- leaves a disabled role instead of a partially provisioned login.
select 'create role leoclean_kb_runtime nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8'
where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_runtime')
\gexec
@ -16,15 +18,19 @@ select 'create role leoclean_kb_stage_owner nologin nosuperuser nocreatedb nocre
where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_stage_owner')
\gexec
select format('alter role leoclean_kb_runtime password %L', :'runtime_password')
\gexec
alter role leoclean_kb_runtime
with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8;
with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8;
alter role leoclean_kb_stage_owner
with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1
password null;
-- Rotate the credential only after an idempotent rerun has disabled any
-- pre-existing LOGIN role. This ordering has an intentional autocommit
-- boundary: interruption can leave NOLOGIN, never a still-enabled old role.
\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD
select format('alter role leoclean_kb_runtime password %L', :'runtime_password')
\gexec
alter role leoclean_kb_runtime reset all;
alter role leoclean_kb_stage_owner reset all;
@ -261,7 +267,7 @@ begin
select 1
from pg_catalog.pg_roles role_row
where role_row.rolname = 'leoclean_kb_runtime'
and (not role_row.rolcanlogin
and (role_row.rolcanlogin
or role_row.rolsuper
or role_row.rolcreatedb
or role_row.rolcreaterole
@ -270,7 +276,7 @@ begin
or role_row.rolbypassrls
or role_row.rolconnlimit <> 8)
) then
raise exception 'leoclean_kb_runtime has unsafe role attributes';
raise exception 'leoclean_kb_runtime has unsafe provisioning role attributes';
end if;
if exists (
@ -818,6 +824,36 @@ commit;
\connect teleo_canonical
-- LOGIN is the last state change. If this verification fails, the transaction
-- rolls back and the role remains NOLOGIN while prior least-privilege grants
-- stay inert for direct authentication.
begin;
alter role leoclean_kb_runtime
with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8;
do $login_verification$
begin
if exists (
select 1
from pg_catalog.pg_roles role_row
where role_row.rolname = 'leoclean_kb_runtime'
and (not role_row.rolcanlogin
or role_row.rolsuper
or role_row.rolcreatedb
or role_row.rolcreaterole
or role_row.rolinherit
or role_row.rolreplication
or role_row.rolbypassrls
or role_row.rolconnlimit <> 8)
) then
raise exception 'leoclean_kb_runtime final login attributes are unsafe';
end if;
end
$login_verification$;
commit;
select jsonb_build_object(
'role', rolname,
'superuser', rolsuper,
@ -835,6 +871,7 @@ select jsonb_build_object(
'can_select_claims', has_table_privilege(rolname, 'public.claims', 'SELECT'),
'can_select_proposals', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'SELECT'),
'can_insert_proposals_directly', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'INSERT'),
'can_connect_legacy_database', has_database_privilege(rolname, 'teleo_kb', 'CONNECT'),
'can_create_database_objects', has_database_privilege(rolname, 'teleo_canonical', 'CREATE'),
'can_create_temp_objects', has_database_privilege(rolname, 'teleo_canonical', 'TEMP'),
'can_stage_proposal', has_function_privilege(

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,219 @@
#!/usr/bin/env python3
"""Verify the effective Leo service environment without exposing its values."""
from __future__ import annotations
import argparse
import json
import sys
from collections.abc import Mapping
from pathlib import Path
from typing import Any
PROJECT_ID = "teleo-501523"
PRIVATE_CLOUDSQL_HOST = "10.61.0.3"
PRIVATE_CLOUDSQL_PORT = "5432"
CANONICAL_DATABASE = "teleo_canonical"
RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime"
SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password"
ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password"
RUNTIME_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
FORBIDDEN_EXACT_FIELDS = frozenset(
{
"ALL_PROXY",
"BASH_ENV",
"BASHOPTS",
"CDPATH",
"CURL_CA_BUNDLE",
"ENV",
"GLOBIGNORE",
"GOOGLE_APPLICATION_CREDENTIALS",
"HTTPS_PROXY",
"HTTP_PROXY",
"IFS",
"NO_PROXY",
"PS4",
"REQUESTS_CA_BUNDLE",
"SHELLOPTS",
"SSL_CERT_DIR",
"SSL_CERT_FILE",
"SSLKEYLOGFILE",
}
)
class VerificationError(RuntimeError):
"""A sanitized failure safe to serialize in a public receipt."""
def __init__(self, code: str, fields: tuple[str, ...] = ()) -> None:
self.code = code
self.fields = tuple(sorted(set(fields)))
super().__init__(code)
def as_dict(self) -> dict[str, object]:
payload: dict[str, object] = {"code": self.code}
if self.fields:
payload["fields"] = list(self.fields)
return payload
class _SanitizedArgumentParser(argparse.ArgumentParser):
def error(self, _message: str) -> None:
raise VerificationError("invalid_arguments")
def validate_pid(value: str | int) -> int:
"""Return a positive process id without retaining the input on failure."""
if isinstance(value, bool) or not isinstance(value, (str, int)):
raise VerificationError("invalid_pid")
if isinstance(value, str) and (not value.isascii() or not value.isdecimal() or value.startswith("0")):
raise VerificationError("invalid_pid")
try:
pid = int(value)
except (TypeError, ValueError):
raise VerificationError("invalid_pid") from None
if pid <= 0:
raise VerificationError("invalid_pid")
return pid
def expected_environment() -> dict[str, str]:
"""Build the exact reviewed environment contract."""
return {
"PATH": RUNTIME_PATH,
"TELEO_CANONICAL_CLOUDSQL_DB": CANONICAL_DATABASE,
"TELEO_CLOUDSQL_DB": CANONICAL_DATABASE,
"TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK": "0",
"TELEO_CLOUDSQL_HOST": PRIVATE_CLOUDSQL_HOST,
"TELEO_CLOUDSQL_PASSWORD_SECRET": SCOPED_PASSWORD_SECRET,
"TELEO_CLOUDSQL_PORT": PRIVATE_CLOUDSQL_PORT,
"TELEO_CLOUDSQL_USER": RUNTIME_DATABASE_ROLE,
"TELEO_GCP_METADATA_ONLY": "1",
"TELEO_GCP_PROJECT": PROJECT_ID,
"TELEO_KB_MODE": "cloudsql",
}
def parse_environment(raw: bytes) -> dict[str, str]:
"""Parse a Linux ``/proc/PID/environ`` block and reject ambiguity."""
if not isinstance(raw, bytes) or not raw or not raw.endswith(b"\0"):
raise VerificationError("environment_malformed")
records = raw[:-1].split(b"\0")
if any(not record for record in records):
raise VerificationError("environment_malformed")
environment: dict[str, str] = {}
try:
for record in records:
encoded_key, separator, encoded_value = record.partition(b"=")
if not separator or not encoded_key:
raise VerificationError("environment_malformed")
key = encoded_key.decode("utf-8", "strict")
value = encoded_value.decode("utf-8", "strict")
if key in environment:
raise VerificationError("environment_malformed")
environment[key] = value
except UnicodeDecodeError:
raise VerificationError("environment_malformed") from None
return environment
def _forbidden_field_labels(environment: Mapping[str, str]) -> tuple[str, ...]:
labels: set[str] = set()
for key in environment:
normalized = key.upper()
if normalized.startswith("PG"):
labels.add("PG*")
if normalized.startswith("CLOUDSDK_"):
labels.add("CLOUDSDK_*")
if normalized.startswith("LD_"):
labels.add("LD_*")
if normalized.startswith("BASH_FUNC_"):
labels.add("BASH_FUNC_*")
if normalized.startswith("PYTHON") and normalized != "PYTHONUNBUFFERED":
labels.add("PYTHON*")
if normalized in FORBIDDEN_EXACT_FIELDS:
labels.add(normalized)
return tuple(sorted(labels))
def validate_environment(environment: Mapping[str, str]) -> tuple[str, ...]:
"""Validate the effective environment and return only reviewed field names."""
if not isinstance(environment, Mapping) or any(
not isinstance(key, str) or not isinstance(value, str) for key, value in environment.items()
):
raise VerificationError("environment_malformed")
expected = expected_environment()
administrator_secret = ADMINISTRATOR_PASSWORD_SECRET.casefold()
if any(administrator_secret in value.casefold() for value in environment.values()):
raise VerificationError("administrator_secret_reference")
forbidden_fields = _forbidden_field_labels(environment)
if forbidden_fields:
raise VerificationError("forbidden_environment_fields", forbidden_fields)
mismatched = tuple(
sorted(key for key, expected_value in expected.items() if environment.get(key) != expected_value)
)
if mismatched:
raise VerificationError("environment_mismatch", mismatched)
return tuple(sorted(expected))
def verify_process_environment(
pid: int,
*,
proc_root: Path = Path("/proc"),
) -> dict[str, object]:
"""Read and verify one process environment, returning a sanitized receipt."""
validated_pid = validate_pid(pid)
raw = (proc_root / str(validated_pid) / "environ").read_bytes()
validated_fields = validate_environment(parse_environment(raw))
return {
"runtime_environment": "scoped",
"status": "pass",
"validated_fields": list(validated_fields),
}
def failure_receipt(error: VerificationError) -> dict[str, object]:
return {"error": error.as_dict(), "status": "fail"}
def canonical_json(payload: Mapping[str, Any]) -> str:
return json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n"
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = _SanitizedArgumentParser(description=__doc__, add_help=False)
parser.add_argument("--pid", required=True)
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
try:
args = parse_args(argv)
receipt = verify_process_environment(validate_pid(args.pid))
returncode = 0
except VerificationError as exc:
receipt = failure_receipt(exc)
returncode = 1
except Exception:
receipt = failure_receipt(VerificationError("internal_error"))
returncode = 1
sys.stdout.write(canonical_json(receipt))
return returncode
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -47,7 +47,7 @@ DEFAULT_HOST = "10.61.0.3"
DEFAULT_PROJECT = "teleo-501523"
DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password"
DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql"
REVIEWED_CLOUDSQL_TOOL_SHA256 = "2617bb501456758131ff41a6c77b3c462e21c28c4348da319b60defb051c7c87"
REVIEWED_CLOUDSQL_TOOL_SHA256 = "66fa84b3bd6fe67e102fbce4509ea28c3537ba5bb1091e1a1a457b7425366064"
REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a"
COUNT_READBACK_RE = re.compile(
r"DB readback:\s*claims:\s*`?(?P<claims>\d+)`?;\s*"

View file

@ -1,9 +1,19 @@
[Service]
UnsetEnvironment=PGPASSWORD
RuntimeDirectory=leoclean-gcp-prod-parallel
RuntimeDirectoryMode=0700
Environment=CLOUDSDK_CONFIG=/run/leoclean-gcp-prod-parallel
UnsetEnvironment=PGPASSFILE PGSERVICE PGHOST PGPORT PGDATABASE PGUSER PGOPTIONS
UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN
UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS CDPATH GLOBIGNORE IFS PS4
UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT LD_DEBUG LD_DEBUG_OUTPUT LD_DYNAMIC_WEAK LD_HWCAP_MASK LD_ORIGIN_PATH LD_PROFILE LD_SHOW_AUXV LD_TRACE_LOADED_OBJECTS LD_USE_LOAD_BIAS LD_VERBOSE LD_WARN
UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT PYTHONUSERBASE PYTHONHTTPSVERIFY
UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy
UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE
ReadOnlyPaths=/home/teleo/.hermes
ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state /home/teleo/.hermes/profiles/leoclean/workspace
BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin
CapabilityBoundingSet=~CAP_SYS_ADMIN
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
Environment=TELEO_KB_MODE=cloudsql
Environment=TELEO_GCP_METADATA_ONLY=1
Environment=TELEO_GCP_PROJECT=teleo-501523
Environment=TELEO_CLOUDSQL_HOST=10.61.0.3
Environment=TELEO_CLOUDSQL_PORT=5432

View file

@ -0,0 +1,797 @@
import hashlib
import json
import os
import stat
import subprocess
from collections.abc import Mapping
from pathlib import Path
import pytest
from ops import verify_gcp_leoclean_runtime_permissions as verifier
RUN_ID = "20260715-leo-security"
SOURCE_REF = f"leo-runtime-permission:{RUN_ID}"
SECRET_VALUE = "unit-only-secret-value%42"
AGENT_ID = "11111111-1111-4111-8111-111111111111"
class FakeRunner:
def __init__(
self,
*,
sqlstate_override: tuple[str, str] | None = None,
sqlstate_missing_for: str | None = None,
unexpected_success_for: str | None = None,
scoped_failure: bool = False,
administrator_mode: str = "denied",
identity_override: Mapping[str, object] | None = None,
role_posture_override: Mapping[str, object] | None = None,
stage_owner_role_posture_override: Mapping[str, object] | None = None,
function_posture_override: Mapping[str, Mapping[str, object]] | None = None,
stage_definition_override: Mapping[str, object] | None = None,
catalog_posture_override: Mapping[str, object] | None = None,
legacy_catalog_posture_override: Mapping[str, object] | None = None,
) -> None:
self.sqlstate_override = sqlstate_override
self.sqlstate_missing_for = sqlstate_missing_for
self.unexpected_success_for = unexpected_success_for
self.scoped_failure = scoped_failure
self.administrator_mode = administrator_mode
self.identity_override = dict(identity_override or {})
self.role_posture_override = dict(role_posture_override or {})
self.stage_owner_role_posture_override = dict(stage_owner_role_posture_override or {})
self.function_posture_override = {
name: dict(values) for name, values in (function_posture_override or {}).items()
}
self.stage_definition_override = dict(stage_definition_override or {})
self.catalog_posture_override = dict(catalog_posture_override or {})
self.legacy_catalog_posture_override = dict(legacy_catalog_posture_override or {})
self.calls: list[tuple[list[str], dict[str, str], int, bool]] = []
self.negative = {check.sql: check for check in verifier._negative_checks(RUN_ID, SOURCE_REF)}
def __call__(
self,
command: list[str],
env: Mapping[str, str],
timeout: int,
discard_stdout: bool,
) -> verifier.CommandResult:
self.calls.append((list(command), dict(env), timeout, discard_stdout))
if command[0] == verifier.GCLOUD_BIN:
scoped = f"--secret={verifier.SCOPED_PASSWORD_SECRET}" in command
if scoped:
if self.scoped_failure:
return verifier.CommandResult(1, SECRET_VALUE, f"scoped failure {SECRET_VALUE}")
return verifier.CommandResult(0, f"{SECRET_VALUE}\n", f"ignored warning {SECRET_VALUE}")
assert f"--secret={verifier.ADMINISTRATOR_PASSWORD_SECRET}" in command
assert discard_stdout is True
if self.administrator_mode == "success":
return verifier.CommandResult(0, SECRET_VALUE, f"ignored {SECRET_VALUE}")
if self.administrator_mode == "wrong-error":
return verifier.CommandResult(1, SECRET_VALUE, f"network unavailable {SECRET_VALUE}")
return verifier.CommandResult(
1,
SECRET_VALUE,
(
"ERROR: PERMISSION_DENIED: Permission "
f"'{verifier.IAM_PERMISSION}' denied for the administrator secret; {SECRET_VALUE}"
),
)
assert command[0] == verifier.PSQL_BIN
assert discard_stdout is False
sql = next(part.removeprefix("--command=") for part in command if part.startswith("--command="))
if sql == verifier._identity_sql():
identity: dict[str, object] = {
"current_user": verifier.RUNTIME_DATABASE_ROLE,
"database": verifier.CANONICAL_DATABASE,
"leak": SECRET_VALUE,
"server_addr": verifier.PRIVATE_CLOUDSQL_HOST,
"server_port": verifier.PRIVATE_CLOUDSQL_PORT,
"session_user": verifier.RUNTIME_DATABASE_ROLE,
"ssl": True,
"ssl_version": "TLSv1.3",
}
identity.update(self.identity_override)
return verifier.CommandResult(0, json.dumps(identity), f"ignored {SECRET_VALUE}")
if sql == verifier._allowed_read_sql():
return verifier.CommandResult(
0,
json.dumps(
{
"claims_rows_sampled": 1,
"leak": SECRET_VALUE,
"proposal_rows_sampled": 0,
}
),
f"ignored {SECRET_VALUE}",
)
if sql == verifier._role_posture_sql():
role_posture: dict[str, object] = {
"bypasses_rls": False,
"can_create_db": False,
"can_create_role": False,
"can_login": True,
"can_replicate": False,
"connection_limit": 8,
"direct_membership_edges": 0,
"inherits_privileges": False,
"is_superuser": False,
"leak": SECRET_VALUE,
"role": verifier.RUNTIME_DATABASE_ROLE,
}
role_posture.update(self.role_posture_override)
return verifier.CommandResult(0, json.dumps(role_posture), f"ignored {SECRET_VALUE}")
if sql == verifier._stage_owner_role_posture_sql():
owner_posture: dict[str, object] = {
"bypasses_rls": False,
"can_create_db": False,
"can_create_role": False,
"can_login": False,
"can_replicate": False,
"connection_limit": -1,
"direct_membership_edges": 0,
"inherits_privileges": False,
"is_superuser": False,
"leak": SECRET_VALUE,
"role": verifier.STAGE_OWNER_DATABASE_ROLE,
}
owner_posture.update(self.stage_owner_role_posture_override)
return verifier.CommandResult(0, json.dumps(owner_posture), f"ignored {SECRET_VALUE}")
if sql == verifier._function_privilege_posture_sql():
function_posture: dict[str, dict[str, object]] = {
name: {
"execute": expected_execute,
"exists": expected_exists,
"leak": SECRET_VALUE,
}
for name, _signature, expected_exists, expected_execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS
}
for name, override in self.function_posture_override.items():
function_posture.setdefault(name, {}).update(override)
function_posture["unexpected_leak"] = {"value": SECRET_VALUE}
return verifier.CommandResult(0, json.dumps(function_posture), f"ignored {SECRET_VALUE}")
if sql == verifier._stage_function_definition_sql():
stage_definition: dict[str, object] = {
"acl_exact": True,
"argument_modes": None,
"configuration": ["search_path=pg_catalog, pg_temp"],
"exists": True,
"leak": SECRET_VALUE,
"runtime_execute": True,
"source": verifier.EXPECTED_STAGE_FUNCTION_SOURCE,
**dict(verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS),
}
stage_definition.update(self.stage_definition_override)
return verifier.CommandResult(0, json.dumps(stage_definition), f"ignored {SECRET_VALUE}")
if sql == verifier._catalog_privilege_posture_sql():
catalog_posture: dict[str, object] = {
field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS
}
catalog_posture.update(
{
"leak": SECRET_VALUE,
**{field: True for field in verifier.CATALOG_TRUE_FIELDS},
}
)
catalog_posture.update(self.catalog_posture_override)
return verifier.CommandResult(0, json.dumps(catalog_posture), f"ignored {SECRET_VALUE}")
if sql == verifier._legacy_catalog_privilege_posture_sql():
legacy_posture: dict[str, object] = {
field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS
}
legacy_posture["leak"] = SECRET_VALUE
legacy_posture.update(self.legacy_catalog_posture_override)
return verifier.CommandResult(0, json.dumps(legacy_posture), f"ignored {SECRET_VALUE}")
if sql == verifier._canary_count_sql(SOURCE_REF):
return verifier.CommandResult(0, "0\n", f"ignored {SECRET_VALUE}")
if sql == verifier._stage_sql(RUN_ID, SOURCE_REF):
return verifier.CommandResult(
0,
json.dumps(
{
"agent_id_matches": True,
"proposal": {
"leak": SECRET_VALUE,
"proposed_by_agent_id": AGENT_ID,
"proposed_by_handle": "leo",
"source_ref": SOURCE_REF,
"status": "pending_review",
},
}
),
f"ignored {SECRET_VALUE}",
)
check = self.negative[sql]
if check.name == self.unexpected_success_for:
return verifier.CommandResult(0, SECRET_VALUE, f"unexpected success {SECRET_VALUE}")
if check.name == self.sqlstate_missing_for:
return verifier.CommandResult(1, SECRET_VALUE, f"denied without state {SECRET_VALUE}")
observed = check.expected_sqlstate
if self.sqlstate_override is not None and check.name == self.sqlstate_override[0]:
observed = self.sqlstate_override[1]
return verifier.CommandResult(
1,
SECRET_VALUE,
f"ERROR: {observed}: expected test denial; {SECRET_VALUE}\nLOCATION: test",
)
def poisoned_environment() -> dict[str, str]:
return {
"CLOUDSDK_AUTH_ACCESS_TOKEN": "must-not-reach-gcloud",
"GOOGLE_APPLICATION_CREDENTIALS": "/tmp/must-not-reach-gcloud.json",
"HOME": "/home/teleo",
"LANG": "C.UTF-8",
"LD_PRELOAD": "/tmp/must-not-load.so",
"OTHER": "preserved",
"PGDATABASE": "wrong-db",
"PGHOST": "public.example.invalid",
"PGPASSWORD": "administrator-password",
"PGSERVICE": "broad-service",
"pgsslmode": "disable",
}
def run_success(runner: FakeRunner) -> dict[str, object]:
return verifier.verify_runtime_permissions(
RUN_ID,
runner=runner,
base_env=poisoned_environment(),
effective_user="teleo",
)
def assert_child_boundaries(runner: FakeRunner) -> None:
assert runner.calls
for command, env, timeout, discard_stdout in runner.calls:
assert SECRET_VALUE not in "\n".join(command)
assert timeout == verifier.COMMAND_TIMEOUT_SECONDS
pg_keys = sorted(key for key in env if key.upper().startswith("PG"))
if command[0] == verifier.PSQL_BIN:
assert pg_keys == ["PGPASSWORD"]
assert env["PGPASSWORD"] == SECRET_VALUE
assert discard_stdout is False
else:
assert pg_keys == []
assert "CLOUDSDK_AUTH_ACCESS_TOKEN" not in env
assert "GOOGLE_APPLICATION_CREDENTIALS" not in env
assert env["HOME"] == "/home/teleo"
assert env["LANG"] == "C"
assert env["LC_ALL"] == "C"
assert env["PATH"] == verifier.CHILD_PATH
assert "LD_PRELOAD" not in env
assert "OTHER" not in env
def test_live_receipt_contract_is_sanitized_and_uses_exact_child_environments() -> None:
runner = FakeRunner()
receipt = run_success(runner)
serialized = verifier.canonical_json(receipt)
assert receipt["status"] == "pass"
assert receipt["current_tier"] == verifier.REQUIRED_TIER
assert receipt["database_identity"] == {
"current_user": verifier.RUNTIME_DATABASE_ROLE,
"database": verifier.CANONICAL_DATABASE,
"server_addr": verifier.PRIVATE_CLOUDSQL_HOST,
"server_port": verifier.PRIVATE_CLOUDSQL_PORT,
"session_user": verifier.RUNTIME_DATABASE_ROLE,
"ssl": True,
"ssl_version": "TLSv1.3",
}
assert receipt["checks"]["rolled_back_proposal"] == {
"agent_id_matches": True,
"proposed_by_agent_id": AGENT_ID,
"proposed_by_handle": "leo",
"source_ref": SOURCE_REF,
"status": "pending_review",
"transaction": "rolled_back",
}
assert receipt["checks"]["canary_rows_before"] == 0
assert receipt["checks"]["canary_rows_after"] == 0
assert receipt["checks"]["role_posture"] == {
"bypasses_rls": False,
"can_create_db": False,
"can_create_role": False,
"can_login": True,
"can_replicate": False,
"connection_limit": 8,
"direct_membership_edges": 0,
"inherits_privileges": False,
"is_superuser": False,
"role": verifier.RUNTIME_DATABASE_ROLE,
}
assert receipt["checks"]["stage_owner_role_posture"] == {
"bypasses_rls": False,
"can_create_db": False,
"can_create_role": False,
"can_login": False,
"can_replicate": False,
"connection_limit": -1,
"direct_membership_edges": 0,
"inherits_privileges": False,
"is_superuser": False,
"role": verifier.STAGE_OWNER_DATABASE_ROLE,
}
assert receipt["checks"]["function_privileges"] == {
name: {
"execute": expected_execute,
"exists": expected_exists,
"signature": signature,
}
for name, signature, expected_exists, expected_execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS
}
assert receipt["checks"]["catalog_privileges"] == {
**{field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS},
**{field: True for field in verifier.CATALOG_TRUE_FIELDS},
}
assert receipt["checks"]["legacy_catalog_privileges"] == {
field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS
}
assert receipt["checks"]["stage_function_definition"] == {
"acl_exact": True,
"argument_modes": None,
"configuration": ["search_path=pg_catalog, pg_temp"],
"exists": True,
**dict(verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS),
"runtime_execute": True,
"signature": verifier.STAGE_FUNCTION_SIGNATURE,
"source_sha256": verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256,
}
assert len(receipt["checks"]["negative_permissions"]) == len(verifier._negative_checks(RUN_ID, SOURCE_REF))
assert receipt["secret_access"]["administrator"]["classification"] == "iam_permission_denied"
assert receipt["secret_access"]["administrator"]["stdout_discarded"] is True
assert SECRET_VALUE not in serialized
assert "administrator-password" not in serialized
assert "stage_leoclean_proposal is restricted" not in serialized
assert_child_boundaries(runner)
administrator_call = runner.calls[-1]
assert administrator_call[0][0] == verifier.GCLOUD_BIN
assert administrator_call[3] is True
@pytest.mark.parametrize(
("attribute", "unsafe_value"),
[
("can_login", False),
("is_superuser", True),
("can_create_db", True),
("can_create_role", True),
("inherits_privileges", True),
("can_replicate", True),
("bypasses_rls", True),
("connection_limit", 7),
],
)
def test_any_unsafe_runtime_role_attribute_fails_closed(attribute: str, unsafe_value: object) -> None:
runner = FakeRunner(role_posture_override={attribute: unsafe_value})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "runtime_role_posture_mismatch"
assert caught.value.check == "runtime_role_posture"
assert SECRET_VALUE not in str(caught.value)
def test_any_direct_runtime_role_membership_edge_fails_closed() -> None:
runner = FakeRunner(role_posture_override={"direct_membership_edges": 1})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "runtime_role_posture_mismatch"
assert caught.value.check == "runtime_role_posture"
@pytest.mark.parametrize(
("attribute", "unsafe_value"),
[
("role", "unexpected-owner"),
("can_login", True),
("is_superuser", True),
("can_create_db", True),
("can_create_role", True),
("inherits_privileges", True),
("can_replicate", True),
("bypasses_rls", True),
("connection_limit", 0),
("direct_membership_edges", 1),
],
)
def test_any_unsafe_stage_owner_role_posture_fails_closed(attribute: str, unsafe_value: object) -> None:
runner = FakeRunner(stage_owner_role_posture_override={attribute: unsafe_value})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "stage_owner_role_posture_mismatch"
assert caught.value.check == "stage_owner_role_posture"
assert SECRET_VALUE not in str(caught.value)
@pytest.mark.parametrize(
"function_name",
["approve_strict_proposal", "assert_approved_proposal", "finish_approved_proposal"],
)
def test_reviewer_or_apply_execute_grant_fails_closed(function_name: str) -> None:
runner = FakeRunner(function_posture_override={function_name: {"execute": True}})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "function_privilege_posture_mismatch"
assert caught.value.check == "function_privilege_posture"
@pytest.mark.parametrize(
"function_name",
[
"stage_leoclean_proposal_5_arg",
"approve_strict_proposal",
"assert_approved_proposal",
"finish_approved_proposal",
],
)
def test_missing_expected_exact_function_fails_closed(function_name: str) -> None:
runner = FakeRunner(function_posture_override={function_name: {"execute": False, "exists": False}})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "function_privilege_posture_mismatch"
assert caught.value.check == "function_privilege_posture"
def test_forged_overload_presence_or_missing_stage_execute_fails_closed() -> None:
forged = FakeRunner(function_posture_override={"stage_leoclean_proposal_6_arg": {"exists": True}})
missing_execute = FakeRunner(function_posture_override={"stage_leoclean_proposal_5_arg": {"execute": False}})
for runner in (forged, missing_execute):
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "function_privilege_posture_mismatch"
def _drifted_value(expected: object) -> object:
if isinstance(expected, bool):
return not expected
if isinstance(expected, int):
return expected + 1
if isinstance(expected, str):
return f"{expected}-drift"
if isinstance(expected, list):
return [*expected, "drift"]
raise AssertionError(f"unsupported test expectation type: {type(expected).__name__}")
@pytest.mark.parametrize(
("field", "unsafe_value"),
[
(field, _drifted_value(expected))
for field, expected in verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS
],
)
def test_any_stage_function_metadata_drift_fails_closed(field: str, unsafe_value: object) -> None:
runner = FakeRunner(stage_definition_override={field: unsafe_value})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "stage_function_definition_mismatch"
assert caught.value.check == "stage_function_definition"
assert SECRET_VALUE not in str(caught.value)
@pytest.mark.parametrize(
("field", "unsafe_value"),
[
("exists", False),
("argument_modes", ["i", "i", "i", "i", "i"]),
("configuration", ["search_path=public"]),
("acl_exact", False),
("runtime_execute", False),
("leakproof", 0),
("argument_defaults", False),
],
)
def test_any_stage_function_contract_drift_fails_closed(field: str, unsafe_value: object) -> None:
runner = FakeRunner(stage_definition_override={field: unsafe_value})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "stage_function_definition_mismatch"
assert caught.value.check == "stage_function_definition"
@pytest.mark.parametrize(
"unsafe_source",
[
None,
42,
verifier.EXPECTED_STAGE_FUNCTION_SOURCE.replace("return to_jsonb(staged);", "return '{}'::jsonb;"),
"x" * (verifier.MAX_STAGE_FUNCTION_SOURCE_BYTES + 1),
],
)
def test_missing_malformed_oversized_or_semantically_drifted_stage_body_fails_closed(
unsafe_source: object,
) -> None:
runner = FakeRunner(stage_definition_override={"source": unsafe_source})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "stage_function_definition_mismatch"
assert caught.value.check == "stage_function_definition"
failure = verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value))
assert "stage_leoclean_proposal is restricted" not in failure
assert SECRET_VALUE not in failure
def test_stage_body_normalization_accepts_only_line_ending_and_framing_newline_changes() -> None:
crlf_source = "\r\n" + verifier.EXPECTED_STAGE_FUNCTION_SOURCE.strip("\n").replace("\n", "\r\n") + "\r\n"
receipt = run_success(FakeRunner(stage_definition_override={"source": crlf_source}))
assert (
receipt["checks"]["stage_function_definition"]["source_sha256"]
== verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256
)
def test_embedded_stage_body_attestation_is_tied_to_the_provisioning_sql_source() -> None:
provisioning_sql = (
Path(__file__).resolve().parents[1] / "ops" / "gcp_leoclean_runtime_role.sql"
).read_text(encoding="utf-8")
function_anchor = "create or replace function kb_stage.stage_leoclean_proposal("
assert provisioning_sql.count(function_anchor) == 1
function_section = provisioning_sql.split(function_anchor, 1)[1]
assert function_section.count("as $function$") == 1
provisioned_source = function_section.split("as $function$", 1)[1].split("$function$;", 1)[0]
assert provisioned_source == verifier.EXPECTED_STAGE_FUNCTION_SOURCE
assert (
hashlib.sha256(verifier.normalize_stage_function_source(provisioned_source).encode("utf-8")).hexdigest()
== verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256
)
def test_catalog_queries_use_exact_regprocedure_signatures_and_both_membership_directions() -> None:
function_sql = verifier._function_privilege_posture_sql()
role_sql = verifier._role_posture_sql()
owner_role_sql = verifier._stage_owner_role_posture_sql()
stage_definition_sql = verifier._stage_function_definition_sql()
catalog_sql = verifier._catalog_privilege_posture_sql()
legacy_sql = verifier._legacy_catalog_privilege_posture_sql()
assert "pg_catalog.to_regprocedure(signature)" in function_sql
for _name, signature, _exists, _execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS:
assert signature in function_sql
assert "membership.roleid = role_row.oid" in role_sql
assert "membership.member = role_row.oid" in role_sql
assert "membership.roleid = role_row.oid" in owner_role_sql
assert "membership.member = role_row.oid" in owner_role_sql
assert verifier.STAGE_OWNER_DATABASE_ROLE in owner_role_sql
assert verifier.STAGE_FUNCTION_SIGNATURE in stage_definition_sql
assert "left join pg_catalog.pg_proc" in stage_definition_sql
assert "function_catalog.prosrc" in stage_definition_sql
assert "function_catalog.prokind" in stage_definition_sql
assert "function_catalog.proargtypes" in stage_definition_sql
assert "function_catalog.prosecdef" in stage_definition_sql
assert "pg_catalog.pg_get_function_result" in stage_definition_sql
assert "pg_catalog.aclexplode" in stage_definition_sql
assert "pg_catalog.pg_shdepend" in catalog_sql
assert "pg_catalog.has_table_privilege" in catalog_sql
assert "pg_catalog.has_column_privilege" in catalog_sql
assert "pg_catalog.has_sequence_privilege" in catalog_sql
assert "pg_catalog.has_function_privilege" in catalog_sql
assert "pg_catalog.aclexplode" in catalog_sql
assert "nspname !~ '^pg_'" in catalog_sql
assert "nspname <> 'information_schema'" in catalog_sql
assert verifier.STAGE_OWNER_DATABASE_ROLE in catalog_sql
assert "teleo_restore" in legacy_sql
assert "pg_catalog.has_schema_privilege" in legacy_sql
assert "pg_catalog.has_table_privilege" in legacy_sql
assert "pg_catalog.has_column_privilege" in legacy_sql
assert "pg_catalog.has_sequence_privilege" in legacy_sql
assert "pg_catalog.has_function_privilege" in legacy_sql
for schema, relation in verifier.READ_TABLE_ALLOWLIST:
assert schema in catalog_sql
assert relation in catalog_sql
@pytest.mark.parametrize("field", verifier.CATALOG_ZERO_COUNT_FIELDS)
def test_any_unexpected_catalog_privilege_fails_closed(field: str) -> None:
runner = FakeRunner(catalog_posture_override={field: 1})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "catalog_privilege_posture_mismatch"
assert caught.value.check == "catalog_privilege_posture"
assert SECRET_VALUE not in str(caught.value)
@pytest.mark.parametrize("field", verifier.CATALOG_TRUE_FIELDS)
def test_any_missing_required_catalog_privilege_fails_closed(field: str) -> None:
runner = FakeRunner(catalog_posture_override={field: False})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "catalog_privilege_posture_mismatch"
@pytest.mark.parametrize("field", verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS)
def test_any_teleo_restore_catalog_privilege_fails_closed(field: str) -> None:
runner = FakeRunner(legacy_catalog_posture_override={field: 1})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "legacy_catalog_privilege_posture_mismatch"
assert caught.value.check == "legacy_catalog_privileges"
assert SECRET_VALUE not in str(caught.value)
def test_teleo_restore_catalog_and_behavioral_denial_are_both_exercised_on_legacy_database() -> None:
runner = FakeRunner()
receipt = run_success(runner)
legacy_catalog_call = next(
command
for command, _env, _timeout, _discard in runner.calls
if any(part == f"--command={verifier._legacy_catalog_privilege_posture_sql()}" for part in command)
)
assert any(f"dbname={verifier.LEGACY_DATABASE}" in part for part in legacy_catalog_call)
legacy_denial = next(
check for check in receipt["checks"]["negative_permissions"] if check["check"] == "legacy_teleo_restore_read"
)
assert legacy_denial == {
"check": "legacy_teleo_restore_read",
"expected_sqlstate": "42501",
"observed_sqlstate": "42501",
"result": "denied",
}
def test_required_sqlstate_mismatch_fails_closed_without_secret_in_error() -> None:
runner = FakeRunner(sqlstate_override=("canonical_insert", "23505"))
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
error = caught.value
assert error.code == "negative_permission_sqlstate_mismatch"
assert error.check == "canonical_insert"
assert error.expected_sqlstate == "42501"
assert error.observed_sqlstate == "23505"
assert SECRET_VALUE not in str(error)
assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, error))
assert_child_boundaries(runner)
def test_missing_verbose_sqlstate_fails_closed_without_raw_stderr() -> None:
runner = FakeRunner(sqlstate_missing_for="forged_proposer_overload")
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
error = caught.value
assert error.code == "negative_permission_sqlstate_missing"
assert error.expected_sqlstate == "42883"
assert error.observed_sqlstate == "missing"
assert SECRET_VALUE not in json.dumps(error.as_dict())
def test_unexpected_permission_success_fails_closed_and_transaction_sql_has_rollback() -> None:
runner = FakeRunner(unexpected_success_for="proposal_update")
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "negative_permission_unexpectedly_succeeded"
assert caught.value.observed_sqlstate == "success"
proposal_update = next(
check for check in verifier._negative_checks(RUN_ID, SOURCE_REF) if check.name == "proposal_update"
)
assert proposal_update.sql.startswith("begin;\n")
assert proposal_update.sql.endswith("\nrollback;")
assert SECRET_VALUE not in str(caught.value)
@pytest.mark.parametrize("administrator_mode", ["success", "wrong-error"])
def test_administrator_secret_must_be_exact_iam_denial_and_stdout_is_discarded(administrator_mode: str) -> None:
runner = FakeRunner(administrator_mode=administrator_mode)
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
expected_code = (
"administrator_secret_unexpectedly_readable"
if administrator_mode == "success"
else "administrator_secret_denial_not_iam_permission"
)
assert caught.value.code == expected_code
assert SECRET_VALUE not in str(caught.value)
assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value))
assert runner.calls[-1][3] is True
def test_scoped_secret_failure_never_repeats_command_output() -> None:
runner = FakeRunner(scoped_failure=True)
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "scoped_secret_access_failed"
assert SECRET_VALUE not in str(caught.value)
assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value))
assert len(runner.calls) == 1
@pytest.mark.parametrize(
"invalid",
["short", "UPPERCASE-RUN", "contains_underscore", "contains.dot", " leading-space", "a" * 65],
)
def test_run_id_validation_is_strict(invalid: str) -> None:
with pytest.raises(ValueError, match="8-64 lowercase"):
verifier.validate_run_id(invalid)
def test_wrong_identity_or_non_ssl_connection_fails_before_permission_probes() -> None:
runner = FakeRunner(identity_override={"ssl": False})
with pytest.raises(verifier.VerificationError) as caught:
run_success(runner)
assert caught.value.code == "private_ssl_identity_mismatch"
assert len(runner.calls) == 2
assert SECRET_VALUE not in str(caught.value)
def test_optional_receipt_is_canonical_mode_0600_and_refuses_symlink(tmp_path: Path) -> None:
payload = {"z": 2, "a": {"safe": True}}
output = tmp_path / "receipt.json"
verifier.write_receipt(output, payload)
assert output.read_text(encoding="utf-8") == verifier.canonical_json(payload)
assert stat.S_IMODE(output.stat().st_mode) == 0o600
assert output.read_text(encoding="utf-8").startswith('{"a":')
target = tmp_path / "target.json"
target.write_text("untouched", encoding="utf-8")
symlink = tmp_path / "symlink.json"
symlink.symlink_to(target)
with pytest.raises(OSError):
verifier.write_receipt(symlink, payload)
assert target.read_text(encoding="utf-8") == "untouched"
def test_subprocess_runner_discards_administrator_stdout_at_file_descriptor(monkeypatch: pytest.MonkeyPatch) -> None:
observed: dict[str, object] = {}
def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
observed["command"] = command
observed.update(kwargs)
return subprocess.CompletedProcess(command, 1, stdout=None, stderr="PERMISSION_DENIED")
monkeypatch.setattr(verifier.subprocess, "run", fake_run)
result = verifier.subprocess_runner([verifier.GCLOUD_BIN, "test"], os.environ, 3, True)
assert observed["stdout"] is subprocess.DEVNULL
assert observed["stdin"] is subprocess.DEVNULL
assert observed["stderr"] is subprocess.PIPE
assert result.stdout == ""
assert result.returncode == 1

View file

@ -0,0 +1,281 @@
import json
import os
import subprocess
import sys
from collections.abc import Mapping
from pathlib import Path
import pytest
from ops import verify_gcp_leoclean_service_environment as verifier
def environment_block(environment: Mapping[str, str]) -> bytes:
return b"\0".join(f"{key}={value}".encode() for key, value in environment.items()) + b"\0"
def test_all_expected_fields_pass_and_receipt_contains_no_values() -> None:
environment = verifier.expected_environment()
environment.update({"HOME": "/home/teleo", "LANG": "C.UTF-8"})
parsed = verifier.parse_environment(environment_block(environment))
validated = verifier.validate_environment(parsed)
receipt = {
"runtime_environment": "scoped",
"status": "pass",
"validated_fields": list(validated),
}
rendered = verifier.canonical_json(receipt)
assert validated == tuple(sorted(verifier.expected_environment()))
assert json.loads(rendered) == receipt
assert rendered == json.dumps(receipt, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n"
for value in verifier.expected_environment().values():
assert value not in rendered
@pytest.mark.parametrize("field", sorted(verifier.expected_environment()))
def test_each_expected_field_mismatch_fails_closed(field: str) -> None:
environment = verifier.expected_environment()
environment[field] = "wrong-value-that-is-not-a-secret"
with pytest.raises(verifier.VerificationError) as caught:
verifier.validate_environment(environment)
assert caught.value.code == "environment_mismatch"
assert caught.value.fields == (field,)
@pytest.mark.parametrize(
("field", "unsafe_value", "expected_code"),
[
("TELEO_CLOUDSQL_USER", "postgres", "environment_mismatch"),
(
"TELEO_CLOUDSQL_PASSWORD_SECRET",
verifier.ADMINISTRATOR_PASSWORD_SECRET,
"administrator_secret_reference",
),
],
)
def test_later_dropin_effective_override_fails_without_outputting_value(
field: str,
unsafe_value: str,
expected_code: str,
) -> None:
environment = verifier.expected_environment()
environment[field] = unsafe_value
with pytest.raises(verifier.VerificationError) as caught:
verifier.validate_environment(environment)
rendered = verifier.canonical_json(verifier.failure_receipt(caught.value))
assert caught.value.code == expected_code
assert unsafe_value not in rendered
@pytest.mark.parametrize(
("field", "safe_label"),
[
("PGPASSWORD", "PG*"),
("pgservice", "PG*"),
("CLOUDSDK_CONFIG", "CLOUDSDK_*"),
("CLOUDSDK_CORE_PROJECT", "CLOUDSDK_*"),
("CLOUDSDK_AUTH_ACCESS_TOKEN", "CLOUDSDK_*"),
("cloudsdk_auth_refresh_token", "CLOUDSDK_*"),
("GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_APPLICATION_CREDENTIALS"),
("LD_PRELOAD", "LD_*"),
("LD_LIBRARY_PATH", "LD_*"),
("ld_audit", "LD_*"),
("BASH_ENV", "BASH_ENV"),
("ENV", "ENV"),
("BASH_FUNC_injected%%", "BASH_FUNC_*"),
("PYTHONHOME", "PYTHON*"),
("PYTHONPATH", "PYTHON*"),
("PYTHONHTTPSVERIFY", "PYTHON*"),
("pythonstartup", "PYTHON*"),
("HTTP_PROXY", "HTTP_PROXY"),
("https_proxy", "HTTPS_PROXY"),
("ALL_PROXY", "ALL_PROXY"),
("no_proxy", "NO_PROXY"),
("SSL_CERT_FILE", "SSL_CERT_FILE"),
("ssl_cert_dir", "SSL_CERT_DIR"),
("REQUESTS_CA_BUNDLE", "REQUESTS_CA_BUNDLE"),
("curl_ca_bundle", "CURL_CA_BUNDLE"),
("SSLKEYLOGFILE", "SSLKEYLOGFILE"),
],
)
def test_environment_file_broader_credentials_and_forbidden_fields_fail_closed(
field: str,
safe_label: str,
) -> None:
environment = verifier.expected_environment()
injected_value = "sensitive-injected-value"
environment[field] = injected_value
with pytest.raises(verifier.VerificationError) as caught:
verifier.validate_environment(environment)
rendered = verifier.canonical_json(verifier.failure_receipt(caught.value))
assert caught.value.code == "forbidden_environment_fields"
assert caught.value.fields == (safe_label,)
assert injected_value not in rendered
def test_arbitrary_forbidden_key_suffix_is_not_an_output_channel() -> None:
environment = verifier.expected_environment()
environment["PG_SECRET_MARKER"] = "not-output"
with pytest.raises(verifier.VerificationError) as caught:
verifier.validate_environment(environment)
rendered = verifier.canonical_json(verifier.failure_receipt(caught.value))
assert "PG_SECRET_MARKER" not in rendered
assert "PG*" in rendered
def test_pythonunbuffered_is_the_only_allowed_python_runtime_tuning() -> None:
environment = verifier.expected_environment()
environment["PYTHONUNBUFFERED"] = "1"
assert verifier.validate_environment(environment) == tuple(sorted(verifier.expected_environment()))
def test_bash_env_executes_before_a_script_but_effective_verifier_rejects_it(tmp_path: Path) -> None:
marker = "BASH_ENV_EXECUTED"
payload = tmp_path / "bash-env.sh"
payload.write_text(f"printf '{marker}\\n'\n", encoding="utf-8")
completed = subprocess.run(
["/bin/bash", "-c", ":"],
env={"BASH_ENV": str(payload), "PATH": verifier.RUNTIME_PATH},
check=False,
capture_output=True,
text=True,
)
assert completed.returncode == 0
assert completed.stdout.strip() == marker
environment = verifier.expected_environment()
environment["BASH_ENV"] = str(payload)
with pytest.raises(verifier.VerificationError) as caught:
verifier.validate_environment(environment)
assert caught.value.fields == ("BASH_ENV",)
def test_admin_secret_reference_in_any_value_fails_without_field_or_value_leak() -> None:
environment = verifier.expected_environment()
environment["UNRELATED"] = f"prefix/{verifier.ADMINISTRATOR_PASSWORD_SECRET}/suffix"
with pytest.raises(verifier.VerificationError) as caught:
verifier.validate_environment(environment)
rendered = verifier.canonical_json(verifier.failure_receipt(caught.value))
assert caught.value.code == "administrator_secret_reference"
assert verifier.ADMINISTRATOR_PASSWORD_SECRET not in rendered
assert "UNRELATED" not in rendered
@pytest.mark.parametrize(
"raw",
[
b"A=1",
b"A=1\0\0",
b"A=1\0BROKEN\0",
b"=value\0",
b"A=1\0A=2\0",
b"A=\xff\0",
b"",
],
)
def test_malformed_nul_environment_is_rejected(raw: bytes) -> None:
with pytest.raises(verifier.VerificationError) as caught:
verifier.parse_environment(raw)
assert caught.value.code == "environment_malformed"
def test_parser_preserves_equals_inside_values() -> None:
assert verifier.parse_environment(b"A=one=two\0B=\0") == {"A": "one=two", "B": ""}
@pytest.mark.parametrize("value", [0, -1, 1.5, "0", "01", "+1", "-3", "not-a-pid", True])
def test_pid_must_be_a_positive_integer(value: object) -> None:
with pytest.raises(verifier.VerificationError) as caught:
verifier.validate_pid(value) # type: ignore[arg-type]
assert caught.value.code == "invalid_pid"
def test_cli_errors_are_canonical_sanitized_json_and_exit_one(capsys: pytest.CaptureFixture[str]) -> None:
secret_input = "not-a-pid-secret-marker"
returncode = verifier.main(["--pid", secret_input])
output = capsys.readouterr()
assert returncode == 1
assert output.err == ""
assert json.loads(output.out) == {"error": {"code": "invalid_pid"}, "status": "fail"}
assert secret_input not in output.out
@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="requires Linux /proc/PID/environ")
def test_cli_reads_real_process_environment_and_detects_effective_conflict(
capsys: pytest.CaptureFixture[str],
) -> None:
expected = verifier.expected_environment()
good_process = subprocess.Popen(
["/bin/sleep", "30"],
env=expected,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
assert verifier.main(["--pid", str(good_process.pid)]) == 0
good_output = capsys.readouterr()
assert good_output.err == ""
assert json.loads(good_output.out)["status"] == "pass"
finally:
good_process.terminate()
good_process.wait(timeout=5)
conflicting = dict(expected)
conflicting["TELEO_CLOUDSQL_USER"] = "postgres"
conflicting["GOOGLE_APPLICATION_CREDENTIALS"] = "/tmp/broader-credential.json"
bad_process = subprocess.Popen(
["/bin/sleep", "30"],
env=conflicting,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
assert verifier.main(["--pid", str(bad_process.pid)]) == 1
bad_output = capsys.readouterr()
receipt = json.loads(bad_output.out)
assert bad_output.err == ""
assert receipt == {
"error": {"code": "forbidden_environment_fields", "fields": ["GOOGLE_APPLICATION_CREDENTIALS"]},
"status": "fail",
}
assert "postgres" not in bad_output.out
assert "/tmp/broader-credential.json" not in bad_output.out
finally:
bad_process.terminate()
bad_process.wait(timeout=5)
def test_unexpected_read_error_is_generic_and_does_not_echo_path(
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
sensitive_marker = "sensitive-proc-read-detail"
def fail_read(_self: object) -> bytes:
raise OSError(sensitive_marker)
monkeypatch.setattr(verifier.Path, "read_bytes", fail_read)
assert verifier.main(["--pid", str(os.getpid())]) == 1
output = capsys.readouterr()
assert json.loads(output.out) == {"error": {"code": "internal_error"}, "status": "fail"}
assert sensitive_marker not in output.out

View file

@ -3,6 +3,7 @@
from __future__ import annotations
import ast
import base64
import contextlib
import importlib.util
import io
@ -64,6 +65,14 @@ def test_cloudsql_wrapper_supports_expected_review_gated_commands() -> None:
assert command in cloudsql_text
assert "status" in vps_text
assert "SCRIPT_DIR=${SCRIPT_PATH%/*}" in wrapper_text
assert 'SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"' in wrapper_text
assert 'CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py"' in wrapper_text
assert wrapper_text.startswith("#!/bin/bash\n")
assert "exec /usr/bin/python3" in wrapper_text
assert "command -v psql" in wrapper_text
assert "gcloud" not in wrapper_text
assert '"gcloud"' not in cloudsql_text
assert "kb_stage.kb_proposals" in cloudsql_text
assert "canonical apply" in cloudsql_text.lower()
@ -95,7 +104,7 @@ def _install_wrapper_fixture(tmp_path: Path, *, missing: frozenset[str] = frozen
target = shutil.which(name)
assert target
(fake_bin / name).symlink_to(target)
for name in ("psql", "gcloud"):
for name in ("psql",):
if name in missing:
continue
executable = fake_bin / name
@ -146,7 +155,7 @@ def test_cloudsql_wrapper_auto_mode_never_falls_back_after_supported_command_fai
assert "local" not in completed.stdout
@pytest.mark.parametrize("missing", ["cloudsql-tool", "psql", "gcloud"])
@pytest.mark.parametrize("missing", ["cloudsql-tool", "psql"])
def test_cloudsql_wrapper_auto_mode_fails_closed_when_prerequisite_is_missing(tmp_path: Path, missing: str) -> None:
wrapper, env = _install_wrapper_fixture(tmp_path, missing=frozenset({missing}))
@ -209,6 +218,72 @@ def test_bridge_source_does_not_commit_raw_secret_values() -> None:
assert not re.search(pattern, combined), pattern
def _runtime_connection_args(module, **overrides):
values = {
"credential_mode": "runtime",
"user": module.RUNTIME_DB_USER,
"password_secret": module.RUNTIME_PASSWORD_SECRET,
"project": module.RUNTIME_GCP_PROJECT,
"host": module.RUNTIME_CLOUDSQL_HOST,
"port": module.RUNTIME_CLOUDSQL_PORT,
"db": module.RUNTIME_CLOUDSQL_DB,
"canonical_db": module.RUNTIME_CLOUDSQL_DB,
"command": "status",
}
values.update(overrides)
return SimpleNamespace(**values)
def _configure_runtime_environment(monkeypatch: pytest.MonkeyPatch) -> None:
for key in tuple(os.environ):
normalized_key = key.upper()
if (
normalized_key.startswith("PG")
or normalized_key.startswith("CLOUDSDK_")
or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS"
or normalized_key in {"LD_PRELOAD", "PYTHONHOME", "PYTHONPATH"}
or normalized_key == "PROXY"
or normalized_key.endswith("_PROXY")
or normalized_key in {
"CURL_CA_BUNDLE",
"PYTHONHTTPSVERIFY",
"REQUESTS_CA_BUNDLE",
"SSL_CERT_DIR",
"SSL_CERT_FILE",
"SSLKEYLOGFILE",
}
):
monkeypatch.delenv(key, raising=False)
monkeypatch.setenv("TELEO_GCP_METADATA_ONLY", "1")
monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0")
class _FakeHttpResponse:
def __init__(self, body: bytes, status: int = 200) -> None:
self.body = body
self.status = status
def __enter__(self):
return self
def __exit__(self, *_args) -> None:
return None
def getcode(self) -> int:
return self.status
def read(self, _limit: int) -> bytes:
return self.body
def _runtime_http_bodies(module, password: bytes = b"scoped-runtime-password") -> list[bytes]:
return [
module.RUNTIME_SERVICE_ACCOUNT.encode("ascii"),
json.dumps({"access_token": "metadata-access-token", "token_type": "Bearer", "expires_in": 300}).encode(),
json.dumps({"payload": {"data": base64.b64encode(password).decode("ascii")}}).encode(),
]
def test_cloudsql_runtime_requires_the_exact_scoped_identity(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
monkeypatch.delenv("PGPASSWORD", raising=False)
@ -249,17 +324,113 @@ def test_cloudsql_runtime_accepts_only_scoped_credentials_and_rejects_inherited_
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = SimpleNamespace(
credential_mode="runtime",
user=module.RUNTIME_DB_USER,
password_secret=module.RUNTIME_PASSWORD_SECRET,
)
monkeypatch.delenv("PGPASSWORD", raising=False)
args = _runtime_connection_args(module)
_configure_runtime_environment(monkeypatch)
module.validate_runtime_connection(args)
monkeypatch.setenv("PGPASSWORD", "")
with pytest.raises(SystemExit, match="Refusing inherited PGPASSWORD"):
with pytest.raises(SystemExit, match="Refusing inherited credential, proxy, TLS, or PostgreSQL environment"):
module.validate_runtime_connection(args)
@pytest.mark.parametrize(
("name", "value"),
[
("GOOGLE_APPLICATION_CREDENTIALS", "/tmp/ambient.json"),
("google_application_credentials", "/tmp/ambient-lowercase.json"),
("CLOUDSDK_CONFIG", "/tmp/gcloud"),
("CLOUDSDK_AUTH_ACCESS_TOKEN", "ambient-token"),
("cloudsdk_core_project", "attacker-project"),
("cloudsdk_auth_access_token", "ambient-lowercase-token"),
("PGHOST", "attacker.invalid"),
("PGOPTIONS", "-c search_path=attacker"),
("pgservice", "attacker-service"),
("ld_preload", "/tmp/attacker.so"),
("LD_LIBRARY_PATH", "/tmp/attacker-library"),
("Ld_AuDiT", "/tmp/attacker-audit.so"),
("BASH_ENV", "/tmp/attacker-bash-env"),
("ENV", "/tmp/attacker-sh-env"),
("BASH_FUNC_psql%%", "() { :; }"),
("PythonHome", "/tmp/attacker-python"),
("pythonpath", "/tmp/attacker-modules"),
("PYTHONHTTPSVERIFY", "0"),
("pythonstartup", "/tmp/attacker-startup.py"),
("https_proxy", "http://attacker.invalid:8080"),
("HtTp_PrOxY", "http://attacker.invalid:8080"),
("ssl_cert_file", "/tmp/attacker.pem"),
("SSL_CERT_DIR", "/tmp/attacker-certs"),
("REQUESTS_CA_BUNDLE", "/tmp/attacker.pem"),
("SSLKEYLOGFILE", "/tmp/attacker-tls-keys"),
],
)
def test_cloudsql_runtime_rejects_ambient_auth_and_pg_environment(
monkeypatch: pytest.MonkeyPatch,
name: str,
value: str,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
_configure_runtime_environment(monkeypatch)
monkeypatch.setenv(name, value)
with pytest.raises(SystemExit, match="Refusing inherited credential, proxy, TLS, or PostgreSQL environment"):
module.validate_runtime_connection(args)
def test_cloudsql_runtime_allows_only_pythonunbuffered_runtime_tuning(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
_configure_runtime_environment(monkeypatch)
monkeypatch.setenv("PYTHONUNBUFFERED", "1")
module.validate_runtime_connection(args)
def test_cloudsql_runtime_requires_explicit_metadata_only_mode(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
_configure_runtime_environment(monkeypatch)
monkeypatch.delenv("TELEO_GCP_METADATA_ONLY")
with pytest.raises(SystemExit, match="TELEO_GCP_METADATA_ONLY=1"):
module.validate_runtime_connection(args)
@pytest.mark.parametrize(
("field", "value", "message"),
[
("project", "other-project", "project=teleo-501523"),
("host", "127.0.0.1", "host=10.61.0.3"),
("port", "6543", "port=5432"),
("db", "teleo_kb", "db=teleo_canonical"),
("canonical_db", "teleo_kb", "canonical_db=teleo_canonical"),
],
)
def test_cloudsql_runtime_rejects_any_target_drift_before_credentials(
monkeypatch: pytest.MonkeyPatch,
field: str,
value: str,
message: str,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module, **{field: value})
monkeypatch.delenv("PGPASSWORD", raising=False)
monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0")
with pytest.raises(SystemExit, match=message):
module.validate_runtime_connection(args)
def test_cloudsql_runtime_rejects_audit_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
monkeypatch.delenv("PGPASSWORD", raising=False)
monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "1")
with pytest.raises(SystemExit, match="AUDIT_FALLBACK=0"):
module.validate_runtime_connection(args)
@ -290,33 +461,44 @@ def test_cloudsql_clone_mode_is_read_only_and_bound_to_one_disposable_database(
module.validate_runtime_connection(args)
def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pytest.MonkeyPatch) -> None:
def test_cloudsql_clone_mode_never_resolves_a_password_secret_argument(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = SimpleNamespace(
credential_mode="runtime",
user=module.RUNTIME_DB_USER,
password_secret=module.RUNTIME_PASSWORD_SECRET,
host="10.61.0.3",
port="5432",
db="teleo_canonical",
canonical_db="teleo_canonical",
project="teleo-501523",
credential_mode="clone-readonly",
user="postgres",
password_secret="gcp-teleo-pgvector-standby-postgres-password",
canonical_db="teleo_clone_canary_123",
db="teleo_clone_canary_123",
command="status",
)
monkeypatch.delenv("PGPASSWORD", raising=False)
monkeypatch.setenv("PGHOST", "attacker.invalid")
monkeypatch.setenv("PGSERVICE", "untrusted-service")
monkeypatch.setenv("PGPASSFILE", "/tmp/untrusted-pgpass")
monkeypatch.setenv("PGOPTIONS", "-c search_path=attacker")
monkeypatch.setenv("PGOPTIONS", module.CLONE_READ_ONLY_PGOPTIONS)
monkeypatch.setattr(
module.subprocess,
"run",
lambda *_args, **_kwargs: pytest.fail("a password-secret argument must never launch a credential helper"),
)
with pytest.raises(SystemExit, match="requires an explicit user and inherited test credential"):
module.password(args)
def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
_configure_runtime_environment(monkeypatch)
monkeypatch.setenv("HOME", "/tmp/untrusted-home")
monkeypatch.setenv("LANG", "attacker-locale")
monkeypatch.setenv("SHELL", "/tmp/untrusted-shell")
monkeypatch.setenv("UNRELATED_AMBIENT_VALUE", "must-not-reach-psql")
captured: dict[str, object] = {}
def fake_run(command, **kwargs):
if command[0] == "gcloud":
return SimpleNamespace(returncode=0, stdout="scoped-runtime-password\n", stderr="")
captured["command"] = command
captured["kwargs"] = kwargs
return SimpleNamespace(returncode=0, stdout="ok\n", stderr="")
monkeypatch.setattr(module, "runtime_password_from_metadata", lambda: "scoped-runtime-password")
monkeypatch.setattr(module.subprocess, "run", fake_run)
assert module.run_psql(args, "select 1;") == "ok\n"
@ -324,16 +506,279 @@ def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pyt
kwargs = captured["kwargs"]
assert isinstance(kwargs, dict)
env = kwargs["env"]
assert env["PGHOST"] == "10.61.0.3"
assert env["PGPORT"] == "5432"
assert env["PGDATABASE"] == "teleo_canonical"
assert env["PGUSER"] == module.RUNTIME_DB_USER
assert env["PGSSLMODE"] == "require"
assert env["PGCONNECT_TIMEOUT"] == "8"
assert env["PGPASSWORD"] == "scoped-runtime-password"
assert "PGOPTIONS" not in env
assert "PGSERVICE" not in env
assert "PGPASSFILE" not in env
assert env == {
"PATH": "/usr/bin:/bin",
"PGHOST": "10.61.0.3",
"PGPORT": "5432",
"PGDATABASE": "teleo_canonical",
"PGUSER": module.RUNTIME_DB_USER,
"PGSSLMODE": "require",
"PGCONNECT_TIMEOUT": "8",
"PGPASSWORD": "scoped-runtime-password",
}
def test_cloudsql_runtime_metadata_and_secret_requests_are_exact_and_silent(
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
_configure_runtime_environment(monkeypatch)
bodies = _runtime_http_bodies(module)
requests: list[tuple[str, str, dict[str, str], float]] = []
def fake_urlopen(request, *, timeout):
requests.append(
(
request.full_url,
request.get_method(),
{key.lower(): value for key, value in request.header_items()},
timeout,
)
)
return _FakeHttpResponse(bodies.pop(0))
monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", fake_urlopen)
assert module.password(args) == "scoped-runtime-password"
assert module.GCE_METADATA_EMAIL_URL == (
"http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email"
)
assert module.GCE_METADATA_TOKEN_URL == (
"http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token"
)
assert requests == [
(
module.GCE_METADATA_EMAIL_URL,
"GET",
{"metadata-flavor": "Google"},
module.HTTP_TIMEOUT_SECONDS,
),
(
module.GCE_METADATA_TOKEN_URL,
"GET",
{"metadata-flavor": "Google"},
module.HTTP_TIMEOUT_SECONDS,
),
(
module.RUNTIME_SECRET_ACCESS_URL,
"GET",
{"accept": "application/json", "authorization": "Bearer metadata-access-token"},
module.HTTP_TIMEOUT_SECONDS,
),
]
assert capsys.readouterr() == ("", "")
def test_cloudsql_runtime_http_opener_disables_proxies_and_redirects() -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
proxy_handlers = [
handler for handler in module.RUNTIME_HTTP_OPENER.handlers if isinstance(handler, module.urllib.request.ProxyHandler)
]
redirect_handlers = [
handler
for handler in module.RUNTIME_HTTP_OPENER.handlers
if isinstance(handler, module.urllib.request.HTTPRedirectHandler)
]
# An empty ProxyHandler suppresses build_opener's environment-backed default;
# urllib omits the empty handler itself from the final chain.
assert proxy_handlers == []
assert module.RUNTIME_PROXY_HANDLER.proxies == {}
assert len(redirect_handlers) == 1
assert type(redirect_handlers[0]) is module._RejectRedirectHandler
def test_cloudsql_runtime_rejects_wrong_metadata_service_account_before_token(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
_configure_runtime_environment(monkeypatch)
calls: list[str] = []
def fake_urlopen(request, *, timeout):
assert timeout == module.HTTP_TIMEOUT_SECONDS
calls.append(request.full_url)
return _FakeHttpResponse(b"unexpected-sa@teleo-501523.iam.gserviceaccount.com")
monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", fake_urlopen)
with pytest.raises(SystemExit, match="did not match the required service account"):
module.password(args)
assert calls == [module.GCE_METADATA_EMAIL_URL]
@pytest.mark.parametrize(
"token_body",
[
b"not-json",
b"[]",
b'{"access_token":"token","token_type":"Basic","expires_in":300}',
b'{"access_token":"token with spaces","token_type":"Bearer","expires_in":300}',
b'{"access_token":"token","token_type":"Bearer","expires_in":0}',
b'{"access_token":"one","access_token":"two","token_type":"Bearer","expires_in":300}',
],
)
def test_cloudsql_runtime_rejects_malformed_metadata_tokens_without_leaking_them(
monkeypatch: pytest.MonkeyPatch,
token_body: bytes,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
_configure_runtime_environment(monkeypatch)
bodies = [module.RUNTIME_SERVICE_ACCOUNT.encode("ascii"), token_body]
monkeypatch.setattr(
module.RUNTIME_HTTP_OPENER,
"open",
lambda _request, *, timeout: _FakeHttpResponse(bodies.pop(0)),
)
with pytest.raises(SystemExit, match="Runtime metadata token response was invalid") as captured:
module.password(args)
assert "token with spaces" not in str(captured.value)
@pytest.mark.parametrize(
"secret_body",
[
b"not-json",
b"[]",
b"{}",
b'{"payload":{"data":"%%%"}}',
json.dumps({"payload": {"data": base64.b64encode(b"x" * 4097).decode("ascii")}}).encode(),
json.dumps({"payload": {"data": base64.b64encode(b"line1\nline2").decode("ascii")}}).encode(),
json.dumps({"payload": {"data": base64.b64encode(b"line1\rline2").decode("ascii")}}).encode(),
json.dumps({"payload": {"data": base64.b64encode(b"nul\x00byte").decode("ascii")}}).encode(),
],
)
def test_cloudsql_runtime_rejects_malformed_secret_payloads_without_leaking_them(
monkeypatch: pytest.MonkeyPatch,
secret_body: bytes,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
_configure_runtime_environment(monkeypatch)
bodies = _runtime_http_bodies(module)
bodies[-1] = secret_body
monkeypatch.setattr(
module.RUNTIME_HTTP_OPENER,
"open",
lambda _request, *, timeout: _FakeHttpResponse(bodies.pop(0)),
)
with pytest.raises(SystemExit, match=r"Scoped runtime secret .* invalid") as captured:
module.password(args)
assert secret_body.decode("utf-8", errors="ignore") not in str(captured.value)
def test_cloudsql_runtime_admin_secret_is_impossible_through_arguments(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module, password_secret="gcp-teleo-pgvector-standby-postgres-password")
_configure_runtime_environment(monkeypatch)
monkeypatch.setattr(
module.RUNTIME_HTTP_OPENER,
"open",
lambda *_args, **_kwargs: pytest.fail("metadata must not be queried for an admin secret argument"),
)
with pytest.raises(SystemExit, match="requires scoped password secret"):
module.password(args)
def test_cloudsql_runtime_http_and_psql_failures_never_echo_credentials(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
_configure_runtime_environment(monkeypatch)
dummy = "DUMMY-RUNTIME-SECRET-DO-NOT-LEAK"
def failed_urlopen(_request, *, timeout):
raise RuntimeError(f"provider echoed {dummy}")
monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", failed_urlopen)
with pytest.raises(SystemExit) as http_error:
module.password(args)
assert dummy not in str(http_error.value)
monkeypatch.setattr(module, "runtime_password_from_metadata", lambda: dummy)
monkeypatch.setattr(
module.subprocess,
"run",
lambda *_args, **_kwargs: SimpleNamespace(returncode=2, stdout=dummy, stderr=dummy),
)
with pytest.raises(SystemExit, match="Cloud SQL query failed with exit status 2") as psql_error:
module.run_psql(args, "select 1;")
assert dummy not in str(psql_error.value)
def test_cloudsql_runtime_read_marker_binds_database_user_private_server_and_ssl(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
marker = {
"database": module.RUNTIME_CLOUDSQL_DB,
"database_user": module.RUNTIME_DB_USER,
"server_addr": module.RUNTIME_CLOUDSQL_HOST,
"server_port": int(module.RUNTIME_CLOUDSQL_PORT),
"ssl": True,
"ssl_version": "TLSv1.3",
"system_identifier": "system-id",
"wal_lsn": "0/1",
}
captured: dict[str, object] = {}
def fake_psql_json_lines(_args, sql, db=None):
captured["sql"] = sql
captured["db"] = db
return [marker]
monkeypatch.setattr(module, "psql_json_lines", fake_psql_json_lines)
assert module.database_read_marker(args) == marker
assert captured["db"] == module.RUNTIME_CLOUDSQL_DB
assert "inet_server_addr()" in str(captured["sql"])
assert "pg_stat_ssl" in str(captured["sql"])
@pytest.mark.parametrize(
("field", "value", "message"),
[
("database", "teleo_kb", "unexpected database identity"),
("database_user", "postgres", "unexpected database identity"),
("server_addr", "127.0.0.1", "unexpected server address"),
("server_port", 6543, "unexpected server port"),
("ssl", False, "encrypted connection"),
],
)
def test_cloudsql_runtime_read_marker_rejects_wrong_live_identity(
monkeypatch: pytest.MonkeyPatch,
field: str,
value: object,
message: str,
) -> None:
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
args = _runtime_connection_args(module)
marker = {
"database": module.RUNTIME_CLOUDSQL_DB,
"database_user": module.RUNTIME_DB_USER,
"server_addr": module.RUNTIME_CLOUDSQL_HOST,
"server_port": int(module.RUNTIME_CLOUDSQL_PORT),
"ssl": True,
"ssl_version": "TLSv1.3",
"system_identifier": "system-id",
"wal_lsn": "0/1",
}
marker[field] = value
monkeypatch.setattr(module, "psql_json_lines", lambda *_args, **_kwargs: [marker])
with pytest.raises(SystemExit, match=message):
module.database_read_marker(args)
def test_cloudsql_proposal_calls_leave_leo_identity_to_the_database(monkeypatch: pytest.MonkeyPatch) -> None:
@ -390,6 +835,7 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None:
assert "can_insert_proposals_directly" in sql
assert "can_create_database_objects" in sql
assert "can_create_temp_objects" in sql
assert "can_connect_legacy_database" in sql
assert "unexpectedly has database CREATE" in sql
assert "REVOKE TEMP FROM PUBLIC would be a" in sql
assert "can execute unexpected SECURITY DEFINER functions" in sql
@ -407,6 +853,15 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None:
)
assert "grant update" not in sql.lower()
assert "grant delete" not in sql.lower()
assert "create role leoclean_kb_runtime nologin" in sql.lower()
disable_statement = "alter role leoclean_kb_runtime\n with nologin nosuperuser"
password_statement = "alter role leoclean_kb_runtime password"
assert sql.lower().index(disable_statement) < sql.lower().index(password_statement)
assert sql.lower().index(disable_statement) < sql.index("\\getenv runtime_password")
assert sql.index("\\getenv runtime_password") < sql.lower().index(password_statement)
assert sql.lower().index("with nologin nosuperuser") < sql.index("\\connect teleo_kb")
assert sql.index("\\connect teleo_kb") < sql.lower().rindex("with login nosuperuser")
assert "leoclean_kb_runtime final login attributes are unsafe" in sql
def _load_module(path: Path):

View file

@ -187,15 +187,60 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped():
assert 'DROPIN_NAME="10-cloudsql-memory.conf"' in script
assert "20-canonical-kb.conf" not in script
assert "sha256sum" in script
assert "--preflight-only" in script
assert "--restart" in script
assert 'python3 "$tool_path" status --format json --redacted' in script
assert '/usr/bin/python3 "$tool_path" status --format json --redacted' in script
assert "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" in script
assert "metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" in script
assert "env -u PGPASSWORD" in script
assert 'run_scoped_status pre "$remote_tmp/$TOOL_REL"' in script
assert 'run_scoped_status post "$profile_bin/cloudsql_memory_tool.py"' in script
assert "169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" in script
assert "ProxyHandler({})" in script
assert "env -i" in script
assert 'run_scoped_status pre "$payload_dir/$TOOL_REL"' in script
assert 'run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL"' in script
assert 'sudo -n -u teleo mkdir -p -- "$receipt_dir"' in script
assert 'if [ "$PREFLIGHT_ONLY" = true ]; then' in script
assert "run_service_wrapper_status post" in script
assert "run_service_wrapper_status verify" in script
assert '"$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted' in script
wrapper_probe = script.split("run_service_wrapper_status() {", maxsplit=1)[1].split(
"\n}\n\nservice_fingerprint", maxsplit=1
)[0]
assert 'nsenter -t "$main_pid" -m --env' in wrapper_probe
assert "/usr/bin/setpriv" in wrapper_probe
assert "--reuid=teleo" in wrapper_probe
assert "TELEO_CLOUDSQL_" not in wrapper_probe
assert "/usr/bin/env -i" not in wrapper_probe
assert 'run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py"' in script
assert 'SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py"' in script
assert 'assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py"' in script
assert 'assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"' in script
assert "REMOTE_CLOUDSDK_CONFIG" not in script
assert "--property=DropInPaths" in script
assert "--property=EnvironmentFiles" in script
assert "Service has an effective EnvironmentFile" in script
assert "Service has a later drop-in" in script
assert "assert_administrator_secret_denied" in script
assert '"stdout_discarded": True' in script
assert '"credential_context": "systemd_main_pid_environment"' in script
administrator_probe = script.split("assert_administrator_secret_denied() {", maxsplit=1)[1].split(
"\n}\n\nrun_scoped_status", maxsplit=1
)[0]
assert 'nsenter -t "$main_pid" -m --env' in administrator_probe
assert "/usr/bin/setpriv" in administrator_probe
assert "env -i" not in administrator_probe
assert 'preflight_service_before="$(service_fingerprint)"' in script
assert 'assert_service_fingerprint "$preflight_service_before"' in script
assert "backup_path" in script
assert "backup_directory_state" in script
assert "restore_path" in script
assert "restore_directory_state" in script
assert 'runtime_grandparent="$(dirname "$runtime_parent")"' in script
assert 'runtime_anchor="$(dirname "$runtime_grandparent")"' in script
assert 'dropin_parent="$(dirname "$dropin_dir")"' in script
assert 'backup_directory_state "$runtime_grandparent" runtime_grandparent' in script
assert 'restore_directory_state "$runtime_grandparent" runtime_grandparent' in script
assert 'sudo test -d "$runtime_anchor"' in script
assert 'sudo test -d "$dropin_parent"' in script
assert 'sudo rmdir -- "$target"' in script
assert "rollback_runtime" in script
assert "trap 'on_exit $?' EXIT" in script
assert 'backup_path "$dropin_dir/$DROPIN_NAME" dropin' in script
@ -205,13 +250,303 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped():
assert 'sub_state" != "running"' in script
assert 'sudo systemctl stop "$SERVICE"' in script
assert 'sudo systemctl start "$SERVICE"' in script
assert 'REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"' in script
assert 'sudo install -d -o root -g root -m 0755 "$runtime_parent"' in script
assert 'sudo install -d -o root -g root -m 0755 "$runtime_bin"' in script
assert 'remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)"' in script
assert 'sudo install -d -o root -g root -m 0755 "$payload_dir"' in script
assert 'sudo install -d -o root -g root -m 0700 "$backup_dir"' in script
assert 'sudo install -d -o teleo -g teleo -m 0700 "$work_dir"' in script
assert "sudo tar --no-same-owner --no-same-permissions" in script
assert 'sudo chown -R root:root "$payload_dir"' in script
assert 'sudo find "$payload_dir" -type d -exec chmod 0555 {} +' in script
assert 'sudo find "$payload_dir" -type l -print -quit' in script
assert 'sudo -n -u teleo test ! -w "$payload_dir"' in script
assert 'sudo -n -u teleo test ! -w "$backup_dir"' in script
assert 'sudo touch "$backup_dir/$name.present"' in script
assert 'sudo test -f "$backup_dir/$name.present"' in script
assert 'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage"' in script
assert 'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage"' in script
assert (
'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" '
'"$permission_verifier_stage"' in script
)
assert (
'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" '
'"$service_env_verifier_stage"' in script
)
assert 'assert_installed_files\nsudo systemctl daemon-reload' in script
assert 'tar -C "$remote_tmp" -xf -' not in script
assert 'mkdir -p "$backup_dir"' not in script
assert 'for protected_file in \\\n "$wrapper" \\' in script
assert '"$permission_verifier" \\' in script
assert '"$service_env_verifier" \\' in script
assert 'sudo -n -u teleo test ! -w "$protected_file"' in script
assert 'for protected_dir in \\\n /usr \\' in script
assert "/usr/local/libexec/livingip" in script
assert 'sudo -n -u teleo test ! -w "$protected_dir"' in script
assert 'for executable_dir in \\\n /usr/local/sbin \\' in script
assert 'sudo stat -Lc \'%U:%G\' "$executable_dir"' in script
assert 'sudo -n -u teleo test ! -w "$executable_dir"' in script
assert 'for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3' in script
assert '/usr/bin/curl -q --noproxy' in script
assert 'test "$(sudo cat "$revision")" = "$SOURCE_REVISION"' in script
assert 'assert_existing_root_directory "$existing_directory"' in script
assert 'assert_existing_root_file "$existing_file"' in script
assert 'validate_restored_path "$runtime_bin/teleo-kb" wrapper' in script
assert 'validate_restored_directory "$runtime_bin" runtime_bin' in script
rollback = script.split("rollback_runtime() {", maxsplit=1)[1].split("\n}\n\non_exit()", maxsplit=1)[0]
assert rollback.index("validate_restored_path") < rollback.index("systemctl daemon-reload")
assert rollback.index("systemctl daemon-reload") < rollback.index('systemctl start "$SERVICE"')
assert rollback.count('if [ "$rollback_rc" -eq 0 ]; then') >= 3
deploy_tail = script.split("mutation_started=true", maxsplit=1)[1]
assert deploy_tail.index("assert_installed_files") < deploy_tail.index("systemctl daemon-reload")
assert deploy_tail.index("systemctl daemon-reload") < deploy_tail.index("assert_unit_configuration")
assert deploy_tail.index("assert_unit_configuration") < deploy_tail.index('systemctl start "$SERVICE"')
for install_line in (
'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL"',
'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL"',
'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL"',
'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL"',
'sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL"',
):
assert install_line in deploy_tail
assert 'sudo install -o root -g root -m 0755 "$work_dir/' not in script
assert 'nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb"' in script
assert 'findmnt -T "$SERVICE_PROFILE_BIN"' in script
assert 'findmnt -M "$SERVICE_HERMES_ROOT"' in script
assert 'findmnt -M "$SERVICE_PROFILE_ROOT/state"' in script
assert 'findmnt -M "$SERVICE_PROFILE_ROOT/workspace"' in script
assert "cap_sys_admin" in script
assert '"current_tier": "T3_live_readonly"' in script
assert '"status": "pass"' in script
assert "UnsetEnvironment=PGPASSWORD" in dropin
assert "RuntimeDirectory=leoclean-gcp-prod-parallel" in dropin
assert "RuntimeDirectoryMode=0700" in dropin
assert "Environment=CLOUDSDK_CONFIG=/run/leoclean-gcp-prod-parallel" in dropin
assert "UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG" in dropin
assert "UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS" in dropin
assert "UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT" in dropin
assert "UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT" in dropin
assert "UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY" in dropin
assert "UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE" in dropin
assert (
"BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin" in dropin
)
assert "ReadOnlyPaths=/home/teleo/.hermes" in dropin
assert (
"ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state "
"/home/teleo/.hermes/profiles/leoclean/workspace" in dropin
)
assert "CapabilityBoundingSet=~CAP_SYS_ADMIN" in dropin
assert "RuntimeDirectory=" not in dropin
assert "Environment=CLOUDSDK_" not in dropin
assert "Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" in dropin
assert "TELEO_KB_MODE=cloudsql" in dropin
assert "TELEO_GCP_METADATA_ONLY=1" in dropin
assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in dropin
assert "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" in dropin
assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in dropin
assert not any(line.startswith("Environment=PGPASSWORD") for line in dropin.splitlines())
assert "gcp-teleo-pgvector-standby-postgres-password" not in dropin
def test_gcp_runtime_embedded_remote_programs_are_bash_syntax_valid() -> None:
script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text()
blocks: dict[str, str] = {}
for variable in ("remote_helpers", "verify_body", "sync_body"):
marker = f"{variable}=$(cat <<'REMOTE'\n"
assert script.count(marker) == 1
start = script.index(marker) + len(marker)
end = script.index("\nREMOTE\n)", start)
blocks[variable] = script[start:end]
for name, body in (
*blocks.items(),
("composed_verify", blocks["remote_helpers"] + "\n" + blocks["verify_body"]),
("composed_sync", blocks["remote_helpers"] + "\n" + blocks["sync_body"]),
):
completed = subprocess.run(
["bash", "-n"],
input=body,
capture_output=True,
check=False,
text=True,
)
assert completed.returncode == 0, f"{name}: {completed.stderr}"
embedded_python = [part.split("\nPY", maxsplit=1)[0] for part in blocks["remote_helpers"].split("<<'PY'\n")[1:]]
assert len(embedded_python) == 2
for index, body in enumerate(embedded_python):
completed = subprocess.run(
["python3", "-c", "import sys; compile(sys.stdin.read(), '<embedded>', 'exec')"],
input=body,
capture_output=True,
check=False,
text=True,
)
assert completed.returncode == 0, f"embedded_python_{index}: {completed.stderr}"
def test_gcp_runtime_rejects_preexisting_symlink_before_mutation(tmp_path: Path) -> None:
script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text()
function_start = script.index("assert_existing_root_directory() {")
function_end = script.index("\n\nbackup_path() {", function_start)
trust_functions = script[function_start:function_end]
real_directory = tmp_path / "real-runtime"
real_directory.mkdir()
symlink = tmp_path / "runtime-link"
symlink.symlink_to(real_directory, target_is_directory=True)
harness = f"""
set -euo pipefail
sudo() {{
if [ "${{1:-}}" = -n ]; then
shift
if [ "${{1:-}}" = -u ]; then shift 2; fi
fi
"$@"
}}
{trust_functions}
assert_existing_root_directory "$1"
"""
completed = subprocess.run(
["bash", "-s", "--", str(symlink)],
input=harness,
capture_output=True,
check=False,
text=True,
)
assert completed.returncode != 0
def test_gcp_runtime_executable_configuration_override_gate() -> None:
script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text()
function_start = script.index("assert_unit_configuration() {")
function_end = script.index("\n\nassert_runtime_environment() {", function_start)
unit_configuration_function = script[function_start:function_end]
harness = f"""
set -euo pipefail
SERVICE=leoclean.service
REMOTE_DROPIN_DIR=/etc/systemd/system/leoclean.service.d
DROPIN_NAME=10-cloudsql-memory.conf
systemctl() {{
case "$*" in
*--property=DropInPaths*) printf '%s\\n' "${{FAKE_DROPINS:-}}" ;;
*--property=EnvironmentFiles*) printf '%s\\n' "${{FAKE_ENVIRONMENT_FILES:-}}" ;;
*) return 90 ;;
esac
}}
{unit_configuration_function}
assert_unit_configuration
"""
reviewed = "/etc/systemd/system/leoclean.service.d/10-cloudsql-memory.conf"
cases = (
{
"FAKE_DROPINS": f"{reviewed} /etc/systemd/system/leoclean.service.d/99-conflict.conf",
"FAKE_ENVIRONMENT_FILES": "",
},
{"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": "/etc/leoclean-runtime.env"},
)
for environment in cases:
completed = subprocess.run(
["bash", "-s"],
input=harness,
capture_output=True,
check=False,
env=environment,
text=True,
)
assert completed.returncode != 0
allowed = subprocess.run(
["bash", "-s"],
input=harness,
capture_output=True,
check=False,
env={"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": ""},
text=True,
)
assert allowed.returncode == 0, allowed.stderr
def test_gcp_runtime_absence_rollback_removes_every_created_directory(tmp_path: Path) -> None:
script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text()
function_start = script.index("backup_directory_state() {")
function_end = script.index("\nrestore_path() {", function_start)
directory_functions = script[function_start:function_end]
harness = f"""
set -euo pipefail
sudo() {{ "$@"; }}
root="$1"
backup_dir="$root/backup"
mkdir -p "$backup_dir" "$root/usr/local" "$root/etc/systemd/system"
runtime_grandparent="$root/usr/local/libexec"
runtime_parent="$runtime_grandparent/livingip"
runtime_bin="$runtime_parent/leoclean-kb"
dropin_dir="$root/etc/systemd/system/service.d"
{directory_functions}
backup_directory_state "$runtime_grandparent" runtime_grandparent
backup_directory_state "$runtime_parent" runtime_parent
backup_directory_state "$runtime_bin" runtime_bin
backup_directory_state "$dropin_dir" dropin_dir
mkdir -p "$runtime_bin" "$dropin_dir"
restore_directory_state "$runtime_bin" runtime_bin
restore_directory_state "$runtime_parent" runtime_parent
restore_directory_state "$runtime_grandparent" runtime_grandparent
restore_directory_state "$dropin_dir" dropin_dir
test ! -e "$runtime_grandparent"
test ! -e "$dropin_dir"
test -d "$root/usr/local"
test -d "$root/etc/systemd/system"
"""
completed = subprocess.run(
["bash", "-s", "--", str(tmp_path)],
input=harness,
capture_output=True,
check=False,
text=True,
)
assert completed.returncode == 0, completed.stderr
def test_gcp_runtime_rollback_never_reloads_or_restarts_after_validation_failure(tmp_path: Path) -> None:
script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text()
function_start = script.index("rollback_runtime() {")
function_end = script.index("\n\non_exit() {", function_start)
rollback_function = script[function_start:function_end]
action_log = tmp_path / "systemctl-actions"
harness = f"""
set -u
SERVICE=leoclean.service
runtime_bin=/usr/local/libexec/livingip/leoclean-kb
runtime_parent=/usr/local/libexec/livingip
runtime_grandparent=/usr/local/libexec
dropin_dir=/etc/systemd/system/leoclean.service.d
DROPIN_NAME=10-cloudsql-memory.conf
action_log="$1"
sudo() {{
if [ "${{1:-}}" = systemctl ]; then
printf '%s\\n' "$2" >>"$action_log"
return 0
fi
"$@"
}}
restore_path() {{ return 0; }}
restore_directory_state() {{ return 0; }}
validate_restored_path() {{ [ "$2" != wrapper ]; }}
validate_restored_directory() {{ return 0; }}
assert_unit_configuration() {{ printf '%s\\n' unit-configuration >>"$action_log"; }}
assert_service_running() {{ printf '%s\\n' assert-running >>"$action_log"; }}
{rollback_function}
rollback_runtime
"""
completed = subprocess.run(
["bash", "-s", "--", str(action_log)],
input=harness,
capture_output=True,
check=False,
text=True,
)
assert completed.returncode != 0
assert action_log.read_text(encoding="utf-8").splitlines() == ["stop"]