fix: enforce scoped GCP Leo runtime credentials
This commit is contained in:
parent
68625b8afa
commit
b7bdcea46a
14 changed files with 5031 additions and 224 deletions
7
.github/workflows/ci.yml
vendored
7
.github/workflows/ci.yml
vendored
|
|
@ -52,6 +52,8 @@ jobs:
|
||||||
ops/restore_gcp_generated_postgres_snapshot.py \
|
ops/restore_gcp_generated_postgres_snapshot.py \
|
||||||
ops/sqlite_to_postgres_dump.py \
|
ops/sqlite_to_postgres_dump.py \
|
||||||
ops/verify_gcp_cloudsql_restore_readback.py \
|
ops/verify_gcp_cloudsql_restore_readback.py \
|
||||||
|
ops/verify_gcp_leoclean_runtime_permissions.py \
|
||||||
|
ops/verify_gcp_leoclean_service_environment.py \
|
||||||
ops/verify_postgres_parity_manifest.py \
|
ops/verify_postgres_parity_manifest.py \
|
||||||
telegram/approvals.py \
|
telegram/approvals.py \
|
||||||
hermes-agent/leoclean-bin/kb_tool.py \
|
hermes-agent/leoclean-bin/kb_tool.py \
|
||||||
|
|
@ -80,6 +82,8 @@ jobs:
|
||||||
tests/test_capture_vps_canonical_postgres_snapshot.py \
|
tests/test_capture_vps_canonical_postgres_snapshot.py \
|
||||||
tests/test_gcp_infra_execute_canary.py \
|
tests/test_gcp_infra_execute_canary.py \
|
||||||
tests/test_gcp_infra_readiness_checker.py \
|
tests/test_gcp_infra_readiness_checker.py \
|
||||||
|
tests/test_gcp_leoclean_runtime_permissions.py \
|
||||||
|
tests/test_gcp_leoclean_service_environment.py \
|
||||||
tests/test_gcp_runtime_baseline_apply.py \
|
tests/test_gcp_runtime_baseline_apply.py \
|
||||||
tests/test_gcp_service_communications.py \
|
tests/test_gcp_service_communications.py \
|
||||||
tests/test_gcp_cloudsql_restore_drill.py \
|
tests/test_gcp_cloudsql_restore_drill.py \
|
||||||
|
|
@ -102,12 +106,15 @@ jobs:
|
||||||
tests/test_restore_gcp_generated_postgres_snapshot.py \
|
tests/test_restore_gcp_generated_postgres_snapshot.py \
|
||||||
tests/test_sqlite_to_postgres_dump.py \
|
tests/test_sqlite_to_postgres_dump.py \
|
||||||
tests/test_sqlite_postgres_restore_canary_capsule.py \
|
tests/test_sqlite_postgres_restore_canary_capsule.py \
|
||||||
|
tests/test_teleo_agent_systemd.py \
|
||||||
tests/test_eval_parse.py \
|
tests/test_eval_parse.py \
|
||||||
tests/test_contributor.py \
|
tests/test_contributor.py \
|
||||||
tests/test_search.py
|
tests/test_search.py
|
||||||
- name: Shell syntax
|
- name: Shell syntax
|
||||||
run: |
|
run: |
|
||||||
bash -n \
|
bash -n \
|
||||||
|
deploy/sync-gcp-leoclean-runtime.sh \
|
||||||
|
hermes-agent/leoclean-bin/teleo-kb \
|
||||||
ops/backup_vps_sqlite_kb.sh \
|
ops/backup_vps_sqlite_kb.sh \
|
||||||
ops/run_gcp_cloudsql_restore_drill.sh \
|
ops/run_gcp_cloudsql_restore_drill.sh \
|
||||||
ops/run_sqlite_postgres_restore_canary.sh
|
ops/run_sqlite_postgres_restore_canary.sh
|
||||||
|
|
|
||||||
|
|
@ -10,23 +10,28 @@ ZONE="${ZONE:-europe-west6-a}"
|
||||||
INSTANCE="${INSTANCE:-teleo-prod-1}"
|
INSTANCE="${INSTANCE:-teleo-prod-1}"
|
||||||
SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}"
|
SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}"
|
||||||
MERGED_REF="${MERGED_REF:-origin/main}"
|
MERGED_REF="${MERGED_REF:-origin/main}"
|
||||||
REMOTE_PROFILE_BIN="${REMOTE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}"
|
REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"
|
||||||
|
SERVICE_PROFILE_BIN="${SERVICE_PROFILE_BIN:-/home/teleo/.hermes/profiles/leoclean/bin}"
|
||||||
|
SERVICE_PROFILE_ROOT="${SERVICE_PROFILE_ROOT:-/home/teleo/.hermes/profiles/leoclean}"
|
||||||
|
SERVICE_HERMES_ROOT="${SERVICE_HERMES_ROOT:-/home/teleo/.hermes}"
|
||||||
REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}"
|
REMOTE_DROPIN_DIR="${REMOTE_DROPIN_DIR:-/etc/systemd/system/$SERVICE.d}"
|
||||||
REMOTE_CLOUDSDK_CONFIG="${REMOTE_CLOUDSDK_CONFIG:-/run/leoclean-gcp-prod-parallel}"
|
|
||||||
EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}"
|
EXPECTED_VM_SERVICE_ACCOUNT="${EXPECTED_VM_SERVICE_ACCOUNT:-sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com}"
|
||||||
|
|
||||||
WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb"
|
WRAPPER_REL="hermes-agent/leoclean-bin/teleo-kb"
|
||||||
TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py"
|
TOOL_REL="hermes-agent/leoclean-bin/cloudsql_memory_tool.py"
|
||||||
|
PERMISSION_VERIFIER_REL="ops/verify_gcp_leoclean_runtime_permissions.py"
|
||||||
|
SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py"
|
||||||
DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf"
|
DROPIN_REL="systemd/leoclean-gcp-prod-parallel-cloudsql.conf"
|
||||||
DROPIN_NAME="10-cloudsql-memory.conf"
|
DROPIN_NAME="10-cloudsql-memory.conf"
|
||||||
|
|
||||||
DRY_RUN=false
|
DRY_RUN=false
|
||||||
RESTART=false
|
RESTART=false
|
||||||
VERIFY_ONLY=false
|
VERIFY_ONLY=false
|
||||||
|
PREFLIGHT_ONLY=false
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
cat <<'USAGE'
|
cat <<'USAGE'
|
||||||
Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--restart] [--verify-only]
|
Usage: deploy/sync-gcp-leoclean-runtime.sh [--dry-run] [--preflight-only] [--restart] [--verify-only]
|
||||||
|
|
||||||
The database role and scoped Secret Manager secret must exist before --restart.
|
The database role and scoped Secret Manager secret must exist before --restart.
|
||||||
This script deploys only the bridge files and non-secret systemd drop-in.
|
This script deploys only the bridge files and non-secret systemd drop-in.
|
||||||
|
|
@ -36,6 +41,7 @@ USAGE
|
||||||
for arg in "$@"; do
|
for arg in "$@"; do
|
||||||
case "$arg" in
|
case "$arg" in
|
||||||
--dry-run) DRY_RUN=true ;;
|
--dry-run) DRY_RUN=true ;;
|
||||||
|
--preflight-only) PREFLIGHT_ONLY=true ;;
|
||||||
--restart) RESTART=true ;;
|
--restart) RESTART=true ;;
|
||||||
--verify-only) VERIFY_ONLY=true ;;
|
--verify-only) VERIFY_ONLY=true ;;
|
||||||
--help|-h)
|
--help|-h)
|
||||||
|
|
@ -50,28 +56,72 @@ for arg in "$@"; do
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
for path in "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; do
|
if $PREFLIGHT_ONLY && $RESTART; then
|
||||||
|
echo "--preflight-only and --restart are mutually exclusive." >&2
|
||||||
|
exit 64
|
||||||
|
fi
|
||||||
|
if $VERIFY_ONLY && { $PREFLIGHT_ONLY || $RESTART; }; then
|
||||||
|
echo "--verify-only cannot be combined with --preflight-only or --restart." >&2
|
||||||
|
exit 64
|
||||||
|
fi
|
||||||
|
|
||||||
|
for path in \
|
||||||
|
"$WRAPPER_REL" \
|
||||||
|
"$TOOL_REL" \
|
||||||
|
"$PERMISSION_VERIFIER_REL" \
|
||||||
|
"$SERVICE_ENV_VERIFIER_REL" \
|
||||||
|
"$DROPIN_REL"; do
|
||||||
if [ ! -f "$REPO_ROOT/$path" ]; then
|
if [ ! -f "$REPO_ROOT/$path" ]; then
|
||||||
echo "Required source file is missing: $path" >&2
|
echo "Required source file is missing: $path" >&2
|
||||||
exit 66
|
exit 66
|
||||||
fi
|
fi
|
||||||
|
if [ -L "$REPO_ROOT/$path" ]; then
|
||||||
|
echo "Refusing symlinked runtime source file: $path" >&2
|
||||||
|
exit 65
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then
|
if ! grep -Fxq 'UnsetEnvironment=PGPASSWORD' "$REPO_ROOT/$DROPIN_REL"; then
|
||||||
echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2
|
echo "Runtime drop-in must explicitly unset inherited PGPASSWORD." >&2
|
||||||
exit 65
|
exit 65
|
||||||
fi
|
fi
|
||||||
if ! grep -Fxq "Environment=CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG" "$REPO_ROOT/$DROPIN_REL"; then
|
if ! grep -Fxq 'Environment=TELEO_GCP_METADATA_ONLY=1' "$REPO_ROOT/$DROPIN_REL"; then
|
||||||
echo "Runtime drop-in CLOUDSDK_CONFIG does not match $REMOTE_CLOUDSDK_CONFIG." >&2
|
echo "Runtime drop-in must enforce metadata-only GCP credentials." >&2
|
||||||
|
exit 65
|
||||||
|
fi
|
||||||
|
if ! grep -Fxq "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$REPO_ROOT/$DROPIN_REL"; then
|
||||||
|
echo "Runtime drop-in does not bind the root-controlled runtime into the service profile." >&2
|
||||||
|
exit 65
|
||||||
|
fi
|
||||||
|
if ! grep -Fxq "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$REPO_ROOT/$DROPIN_REL"; then
|
||||||
|
echo "Runtime drop-in does not protect the service-profile ancestor from replacement." >&2
|
||||||
|
exit 65
|
||||||
|
fi
|
||||||
|
if ! grep -Fxq "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$REPO_ROOT/$DROPIN_REL"; then
|
||||||
|
echo "Runtime drop-in does not limit service-profile writes to state and workspace." >&2
|
||||||
|
exit 65
|
||||||
|
fi
|
||||||
|
if ! grep -Fxq 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$REPO_ROOT/$DROPIN_REL"; then
|
||||||
|
echo "Runtime drop-in must remove CAP_SYS_ADMIN from the service." >&2
|
||||||
exit 65
|
exit 65
|
||||||
fi
|
fi
|
||||||
|
|
||||||
SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)"
|
SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)"
|
||||||
if ! git -C "$REPO_ROOT" diff --quiet HEAD -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL"; then
|
if ! git -C "$REPO_ROOT" diff --quiet HEAD -- \
|
||||||
|
"$WRAPPER_REL" \
|
||||||
|
"$TOOL_REL" \
|
||||||
|
"$PERMISSION_VERIFIER_REL" \
|
||||||
|
"$SERVICE_ENV_VERIFIER_REL" \
|
||||||
|
"$DROPIN_REL"; then
|
||||||
echo "Refusing to deploy runtime files that do not match HEAD." >&2
|
echo "Refusing to deploy runtime files that do not match HEAD." >&2
|
||||||
exit 65
|
exit 65
|
||||||
fi
|
fi
|
||||||
if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- "$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL")" ]; then
|
if [ -n "$(git -C "$REPO_ROOT" ls-files --others --exclude-standard -- \
|
||||||
|
"$WRAPPER_REL" \
|
||||||
|
"$TOOL_REL" \
|
||||||
|
"$PERMISSION_VERIFIER_REL" \
|
||||||
|
"$SERVICE_ENV_VERIFIER_REL" \
|
||||||
|
"$DROPIN_REL")" ]; then
|
||||||
echo "Refusing to deploy untracked runtime files." >&2
|
echo "Refusing to deploy untracked runtime files." >&2
|
||||||
exit 65
|
exit 65
|
||||||
fi
|
fi
|
||||||
|
|
@ -85,6 +135,8 @@ if ! git -C "$REPO_ROOT" merge-base --is-ancestor "$SOURCE_REVISION" "$MERGED_RE
|
||||||
fi
|
fi
|
||||||
WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')"
|
WRAPPER_SHA="$(shasum -a 256 "$REPO_ROOT/$WRAPPER_REL" | awk '{print $1}')"
|
||||||
TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')"
|
TOOL_SHA="$(shasum -a 256 "$REPO_ROOT/$TOOL_REL" | awk '{print $1}')"
|
||||||
|
PERMISSION_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$PERMISSION_VERIFIER_REL" | awk '{print $1}')"
|
||||||
|
SERVICE_ENV_VERIFIER_SHA="$(shasum -a 256 "$REPO_ROOT/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')"
|
||||||
DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')"
|
DROPIN_SHA="$(shasum -a 256 "$REPO_ROOT/$DROPIN_REL" | awk '{print $1}')"
|
||||||
|
|
||||||
GCP_SSH=(
|
GCP_SSH=(
|
||||||
|
|
@ -123,64 +175,132 @@ assert_service_running() {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
assert_runtime_environment() {
|
assert_unit_configuration() {
|
||||||
local environment runtime_dir_state
|
local require_reviewed="${1:-true}"
|
||||||
environment="$(systemctl show "$SERVICE" --property=Environment --value)"
|
local dropin_path dropin_name dropin_paths environment_files reviewed_dropin_seen=false
|
||||||
case " $environment " in
|
dropin_paths="$(systemctl show "$SERVICE" --property=DropInPaths --value)"
|
||||||
*" PGPASSWORD="*)
|
environment_files="$(systemctl show "$SERVICE" --property=EnvironmentFiles --value)"
|
||||||
echo "Service environment still contains PGPASSWORD." >&2
|
if [ -n "$environment_files" ]; then
|
||||||
|
echo "Service has an effective EnvironmentFile, which is not allowed for the scoped runtime." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
for dropin_path in $dropin_paths; do
|
||||||
|
if [ "$dropin_path" = "$REMOTE_DROPIN_DIR/$DROPIN_NAME" ]; then
|
||||||
|
reviewed_dropin_seen=true
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
dropin_name="${dropin_path##*/}"
|
||||||
|
if [[ "$dropin_name" > "$DROPIN_NAME" ]]; then
|
||||||
|
echo "Service has a later drop-in than the reviewed Cloud SQL configuration: $dropin_name" >&2
|
||||||
return 1
|
return 1
|
||||||
;;
|
fi
|
||||||
esac
|
done
|
||||||
case " $environment " in
|
if [ "$require_reviewed" = true ] && [ "$reviewed_dropin_seen" != true ]; then
|
||||||
*" CLOUDSDK_CONFIG=$REMOTE_CLOUDSDK_CONFIG "*) ;;
|
echo "Service does not load the reviewed Cloud SQL drop-in." >&2
|
||||||
*)
|
|
||||||
echo "Service does not use the isolated CLOUDSDK_CONFIG." >&2
|
|
||||||
return 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
runtime_dir_state="$(stat -c '%U:%G:%a' "$REMOTE_CLOUDSDK_CONFIG")"
|
|
||||||
if [ "$runtime_dir_state" != "teleo:teleo:700" ]; then
|
|
||||||
echo "Isolated CLOUDSDK_CONFIG has unexpected ownership or mode: $runtime_dir_state" >&2
|
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
assert_runtime_environment() {
|
||||||
|
local verifier_path="$1" require_reviewed="${2:-true}" main_pid
|
||||||
|
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
|
||||||
|
sudo /usr/bin/python3 "$verifier_path" --pid "$main_pid"
|
||||||
|
assert_unit_configuration "$require_reviewed"
|
||||||
|
}
|
||||||
|
|
||||||
assert_attached_service_account() {
|
assert_attached_service_account() {
|
||||||
local sdk_config="$1" metadata_email
|
local metadata_email
|
||||||
sudo install -d -o teleo -g teleo -m 0700 "$sdk_config"
|
|
||||||
metadata_email="$(
|
metadata_email="$(
|
||||||
sudo -n -u teleo env -u PGPASSWORD \
|
sudo -n -u teleo env -i \
|
||||||
HOME=/home/teleo \
|
HOME=/home/teleo \
|
||||||
CLOUDSDK_CONFIG="$sdk_config" \
|
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||||
curl --fail --silent --show-error \
|
/usr/bin/curl -q --noproxy '*' --proto '=http' --max-time 10 --fail --silent --show-error \
|
||||||
--header 'Metadata-Flavor: Google' \
|
--header 'Metadata-Flavor: Google' \
|
||||||
'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email'
|
'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email'
|
||||||
)"
|
)"
|
||||||
if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then
|
if [ "$metadata_email" != "$EXPECTED_VM_SERVICE_ACCOUNT" ]; then
|
||||||
echo "Attached VM service account mismatch: $metadata_email" >&2
|
echo "Attached VM service account mismatch: $metadata_email" >&2
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# An empty, phase-local Cloud SDK config forces this token request to use
|
|
||||||
# the VM's attached identity instead of an operator's cached credentials.
|
|
||||||
sudo -n -u teleo env -u PGPASSWORD \
|
|
||||||
HOME=/home/teleo \
|
|
||||||
CLOUDSDK_CONFIG="$sdk_config" \
|
|
||||||
CLOUDSDK_CORE_PROJECT="$PROJECT" \
|
|
||||||
gcloud --quiet auth print-access-token >/dev/null
|
|
||||||
echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email"
|
echo "ATTACHED_VM_SERVICE_ACCOUNT=$metadata_email"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
assert_administrator_secret_denied() {
|
||||||
|
local main_pid
|
||||||
|
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
|
||||||
|
sudo nsenter -t "$main_pid" -m --env -- \
|
||||||
|
/usr/bin/setpriv \
|
||||||
|
--reuid=teleo \
|
||||||
|
--regid=teleo \
|
||||||
|
--init-groups \
|
||||||
|
--no-new-privs \
|
||||||
|
/usr/bin/python3 - "$PROJECT" "$EXPECTED_VM_SERVICE_ACCOUNT" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
project, expected_email = sys.argv[1:]
|
||||||
|
metadata_headers = {"Metadata-Flavor": "Google"}
|
||||||
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||||
|
|
||||||
|
|
||||||
|
def metadata(path: str) -> bytes:
|
||||||
|
request = urllib.request.Request(
|
||||||
|
"http://169.254.169.254/computeMetadata/v1/" + path,
|
||||||
|
headers=metadata_headers,
|
||||||
|
)
|
||||||
|
with opener.open(request, timeout=10) as response:
|
||||||
|
return response.read(65536)
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
email = metadata("instance/service-accounts/default/email").decode("utf-8").strip()
|
||||||
|
if email != expected_email:
|
||||||
|
raise RuntimeError
|
||||||
|
token_payload = json.loads(metadata("instance/service-accounts/default/token"))
|
||||||
|
token = token_payload.get("access_token")
|
||||||
|
if token_payload.get("token_type") != "Bearer" or not isinstance(token, str) or not token:
|
||||||
|
raise RuntimeError
|
||||||
|
secret = urllib.parse.quote("gcp-teleo-pgvector-standby-postgres-password", safe="")
|
||||||
|
url = f"https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{secret}/versions/latest:access"
|
||||||
|
request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
|
||||||
|
try:
|
||||||
|
response = opener.open(request, timeout=10)
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
body = error.read(65536)
|
||||||
|
parsed = json.loads(body)
|
||||||
|
detail = parsed.get("error", {})
|
||||||
|
message = str(detail.get("message", "")).lower()
|
||||||
|
if error.code != 403 or detail.get("status") != "PERMISSION_DENIED":
|
||||||
|
raise RuntimeError from None
|
||||||
|
if "secretmanager.versions.access" not in message:
|
||||||
|
raise RuntimeError
|
||||||
|
else:
|
||||||
|
response.close()
|
||||||
|
raise RuntimeError
|
||||||
|
except Exception:
|
||||||
|
raise SystemExit("administrator secret was not the exact expected metadata-identity IAM denial") from None
|
||||||
|
|
||||||
|
print(json.dumps({
|
||||||
|
"administrator_secret_access": "iam_permission_denied",
|
||||||
|
"credential_context": "systemd_main_pid_environment",
|
||||||
|
"identity_source": "gce_metadata",
|
||||||
|
"stdout_discarded": True,
|
||||||
|
}, sort_keys=True))
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
run_scoped_status() {
|
run_scoped_status() {
|
||||||
local phase="$1" tool_path="$2" sdk_config="$3"
|
local phase="$1" tool_path="$2"
|
||||||
assert_attached_service_account "$sdk_config"
|
assert_attached_service_account
|
||||||
echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical"
|
echo "SCOPED_STATUS phase=$phase user=teleo database=teleo_canonical"
|
||||||
sudo -n -u teleo env -u PGPASSWORD \
|
sudo -n -u teleo env -i \
|
||||||
HOME=/home/teleo \
|
HOME=/home/teleo \
|
||||||
CLOUDSDK_CONFIG="$sdk_config" \
|
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||||
CLOUDSDK_CORE_PROJECT="$PROJECT" \
|
|
||||||
TELEO_GCP_PROJECT=teleo-501523 \
|
TELEO_GCP_PROJECT=teleo-501523 \
|
||||||
|
TELEO_GCP_METADATA_ONLY=1 \
|
||||||
TELEO_CLOUDSQL_HOST=10.61.0.3 \
|
TELEO_CLOUDSQL_HOST=10.61.0.3 \
|
||||||
TELEO_CLOUDSQL_PORT=5432 \
|
TELEO_CLOUDSQL_PORT=5432 \
|
||||||
TELEO_CLOUDSQL_DB=teleo_canonical \
|
TELEO_CLOUDSQL_DB=teleo_canonical \
|
||||||
|
|
@ -188,46 +308,238 @@ run_scoped_status() {
|
||||||
TELEO_CLOUDSQL_USER=leoclean_kb_runtime \
|
TELEO_CLOUDSQL_USER=leoclean_kb_runtime \
|
||||||
TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \
|
TELEO_CLOUDSQL_PASSWORD_SECRET=gcp-teleo-pgvector-standby-leoclean-kb-runtime-password \
|
||||||
TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \
|
TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0 \
|
||||||
python3 "$tool_path" status --format json --redacted
|
/usr/bin/python3 "$tool_path" status --format json --redacted
|
||||||
|
}
|
||||||
|
|
||||||
|
run_service_wrapper_status() {
|
||||||
|
local phase="$1" main_pid
|
||||||
|
assert_attached_service_account
|
||||||
|
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
|
||||||
|
echo "SCOPED_WRAPPER_STATUS phase=$phase user=teleo database=teleo_canonical"
|
||||||
|
# --env imports the target MainPID's effective /proc environment. setpriv
|
||||||
|
# drops back to the service's Unix identity without sudo resetting that env.
|
||||||
|
sudo nsenter -t "$main_pid" -m --env -- \
|
||||||
|
/usr/bin/setpriv \
|
||||||
|
--reuid=teleo \
|
||||||
|
--regid=teleo \
|
||||||
|
--init-groups \
|
||||||
|
--no-new-privs \
|
||||||
|
"$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted
|
||||||
|
}
|
||||||
|
|
||||||
|
service_fingerprint() {
|
||||||
|
printf '%s|%s|%s|%s' \
|
||||||
|
"$(systemctl show "$SERVICE" --property=ActiveState --value)" \
|
||||||
|
"$(systemctl show "$SERVICE" --property=SubState --value)" \
|
||||||
|
"$(systemctl show "$SERVICE" --property=MainPID --value)" \
|
||||||
|
"$(systemctl show "$SERVICE" --property=NRestarts --value)"
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_service_fingerprint() {
|
||||||
|
local expected="$1" actual
|
||||||
|
actual="$(service_fingerprint)"
|
||||||
|
if [ "$actual" != "$expected" ]; then
|
||||||
|
echo "Service state changed during the read-only preflight." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "PREFLIGHT_SERVICE_UNCHANGED fingerprint=$actual"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_permission_verifier() {
|
||||||
|
local phase="$1" verifier_path="$2" receipt_path="$3" receipt_dir run_id receipt_state
|
||||||
|
run_id="$(date -u +%Y%m%d%H%M%S)-$phase"
|
||||||
|
receipt_dir="$(dirname "$receipt_path")"
|
||||||
|
sudo -n -u teleo mkdir -p -- "$receipt_dir"
|
||||||
|
sudo -n -u teleo chmod 0700 "$receipt_dir"
|
||||||
|
echo "NEGATIVE_PERMISSION_RECEIPT phase=$phase run_id=$run_id"
|
||||||
|
sudo -n -u teleo env -i \
|
||||||
|
HOME=/home/teleo \
|
||||||
|
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||||
|
/usr/bin/python3 "$verifier_path" --run-id "$run_id" --output "$receipt_path"
|
||||||
|
receipt_state="$(sudo stat -c '%U:%G:%a' "$receipt_path")"
|
||||||
|
if [ "$receipt_state" != "teleo:teleo:600" ]; then
|
||||||
|
echo "Permission receipt has unexpected ownership or mode: $receipt_state" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
sudo /usr/bin/python3 - "$receipt_path" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
receipt = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
|
||||||
|
expected = {
|
||||||
|
"artifact": "gcp_leoclean_runtime_permissions",
|
||||||
|
"current_tier": "T3_live_readonly",
|
||||||
|
"mode": "live_private_gcp_staging",
|
||||||
|
"required_tier": "T3_live_readonly",
|
||||||
|
"status": "pass",
|
||||||
|
}
|
||||||
|
mismatched = sorted(key for key, value in expected.items() if receipt.get(key) != value)
|
||||||
|
if mismatched:
|
||||||
|
raise SystemExit("permission receipt contract mismatch: " + ",".join(mismatched))
|
||||||
|
print(json.dumps({"permission_receipt": "validated", "validated_fields": sorted(expected)}, sort_keys=True))
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_installed_files() {
|
||||||
|
local wrapper tool permission_verifier service_env_verifier dropin revision executable_dir protected_dir protected_file
|
||||||
|
wrapper="$REMOTE_RUNTIME_BIN/teleo-kb"
|
||||||
|
tool="$REMOTE_RUNTIME_BIN/cloudsql_memory_tool.py"
|
||||||
|
permission_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py"
|
||||||
|
service_env_verifier="$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"
|
||||||
|
dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME"
|
||||||
|
revision="$REMOTE_RUNTIME_BIN/.livingip-runtime-revision"
|
||||||
|
test "$(sudo sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA"
|
||||||
|
test "$(sudo sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA"
|
||||||
|
test "$(sudo sha256sum "$permission_verifier" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
|
||||||
|
test "$(sudo sha256sum "$service_env_verifier" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
|
||||||
|
test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$REMOTE_RUNTIME_BIN")" = "root:root:755"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$wrapper")" = "root:root:755"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$tool")" = "root:root:755"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$permission_verifier")" = "root:root:755"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$service_env_verifier")" = "root:root:755"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$revision")" = "root:root:644"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$dropin")" = "root:root:644"
|
||||||
|
test "$(sudo cat "$revision")" = "$SOURCE_REVISION"
|
||||||
|
for protected_dir in \
|
||||||
|
/usr \
|
||||||
|
/usr/local \
|
||||||
|
/usr/local/libexec \
|
||||||
|
/usr/local/libexec/livingip \
|
||||||
|
"$REMOTE_RUNTIME_BIN" \
|
||||||
|
"$REMOTE_DROPIN_DIR"; do
|
||||||
|
sudo test -d "$protected_dir"
|
||||||
|
sudo test ! -L "$protected_dir"
|
||||||
|
test "$(sudo stat -c '%U:%G' "$protected_dir")" = "root:root"
|
||||||
|
sudo -n -u teleo test ! -w "$protected_dir"
|
||||||
|
done
|
||||||
|
for protected_file in \
|
||||||
|
"$wrapper" \
|
||||||
|
"$tool" \
|
||||||
|
"$permission_verifier" \
|
||||||
|
"$service_env_verifier" \
|
||||||
|
"$revision" \
|
||||||
|
"$dropin"; do
|
||||||
|
sudo test -f "$protected_file"
|
||||||
|
sudo test ! -L "$protected_file"
|
||||||
|
sudo -n -u teleo test ! -w "$protected_file"
|
||||||
|
done
|
||||||
|
for executable_dir in \
|
||||||
|
/usr/local/sbin \
|
||||||
|
/usr/local/bin \
|
||||||
|
/usr/sbin \
|
||||||
|
/usr/bin \
|
||||||
|
/sbin \
|
||||||
|
/bin; do
|
||||||
|
sudo test -d "$executable_dir"
|
||||||
|
test "$(sudo stat -Lc '%U:%G' "$executable_dir")" = "root:root"
|
||||||
|
sudo -n -u teleo test ! -w "$executable_dir"
|
||||||
|
done
|
||||||
|
for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3; do
|
||||||
|
sudo test -x "$protected_file"
|
||||||
|
test "$(sudo stat -Lc '%U:%G' "$protected_file")" = "root:root"
|
||||||
|
sudo -n -u teleo test ! -w "$protected_file"
|
||||||
|
done
|
||||||
|
sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin"
|
||||||
|
sudo grep -E '^Environment=TELEO_(KB_MODE|GCP_PROJECT|GCP_METADATA_ONLY|CLOUDSQL_|CANONICAL_CLOUDSQL_DB)' "$dropin"
|
||||||
|
sudo grep -Fx "ReadOnlyPaths=$SERVICE_HERMES_ROOT" "$dropin"
|
||||||
|
sudo grep -Fx "ReadWritePaths=$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" "$dropin"
|
||||||
|
sudo grep -Fx "BindReadOnlyPaths=$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" "$dropin"
|
||||||
|
sudo grep -Fx 'CapabilityBoundingSet=~CAP_SYS_ADMIN' "$dropin"
|
||||||
}
|
}
|
||||||
|
|
||||||
assert_deployed_files() {
|
assert_deployed_files() {
|
||||||
local wrapper tool dropin
|
local main_pid namespace_state source_state mount_state
|
||||||
wrapper="$REMOTE_PROFILE_BIN/teleo-kb"
|
local hermes_mount_state state_mount_state workspace_mount_state capability_set
|
||||||
tool="$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py"
|
assert_installed_files
|
||||||
dropin="$REMOTE_DROPIN_DIR/$DROPIN_NAME"
|
|
||||||
test "$(sha256sum "$wrapper" | awk '{print $1}')" = "$WRAPPER_SHA"
|
main_pid="$(systemctl show "$SERVICE" --property=MainPID --value)"
|
||||||
test "$(sha256sum "$tool" | awk '{print $1}')" = "$TOOL_SHA"
|
source_state="$(stat -Lc '%d:%i' "$REMOTE_RUNTIME_BIN")"
|
||||||
test "$(sudo sha256sum "$dropin" | awk '{print $1}')" = "$DROPIN_SHA"
|
namespace_state="$(sudo nsenter -t "$main_pid" -m -- stat -Lc '%d:%i' "$SERVICE_PROFILE_BIN")"
|
||||||
sudo grep -Fx 'UnsetEnvironment=PGPASSWORD' "$dropin"
|
if [ "$namespace_state" != "$source_state" ]; then
|
||||||
sudo grep -E '^Environment=(CLOUDSDK_CONFIG|TELEO_(KB_MODE|GCP_PROJECT|CLOUDSQL_|CANONICAL_CLOUDSQL_DB))' "$dropin"
|
echo "Service namespace does not bind the reviewed runtime directory." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -T "$SERVICE_PROFILE_BIN" -n -o OPTIONS)"
|
||||||
|
case ",$mount_state," in
|
||||||
|
*,ro,*) ;;
|
||||||
|
*)
|
||||||
|
echo "Service runtime bind is not read-only." >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
hermes_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_HERMES_ROOT" -n -o OPTIONS)"
|
||||||
|
state_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/state" -n -o OPTIONS)"
|
||||||
|
workspace_mount_state="$(sudo nsenter -t "$main_pid" -m -- findmnt -M "$SERVICE_PROFILE_ROOT/workspace" -n -o OPTIONS)"
|
||||||
|
case ",$hermes_mount_state," in
|
||||||
|
*,ro,*) ;;
|
||||||
|
*)
|
||||||
|
echo "Service .hermes ancestor mount is not read-only." >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
case ",$state_mount_state," in
|
||||||
|
*,rw,*) ;;
|
||||||
|
*)
|
||||||
|
echo "Service state mount is not writable." >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
case ",$workspace_mount_state," in
|
||||||
|
*,rw,*) ;;
|
||||||
|
*)
|
||||||
|
echo "Service workspace mount is not writable." >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w /home
|
||||||
|
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT"
|
||||||
|
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_HERMES_ROOT/profiles"
|
||||||
|
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_ROOT"
|
||||||
|
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test ! -w "$SERVICE_PROFILE_BIN"
|
||||||
|
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/state"
|
||||||
|
sudo nsenter -t "$main_pid" -m -- /usr/bin/sudo -n -u teleo test -w "$SERVICE_PROFILE_ROOT/workspace"
|
||||||
|
capability_set="$(systemctl show "$SERVICE" --property=CapabilityBoundingSet --value)"
|
||||||
|
if printf '%s\n' "$capability_set" | grep -Eqi '(^|[[:space:]])cap_sys_admin([[:space:]]|$)'; then
|
||||||
|
echo "Service capability bounding set still contains CAP_SYS_ADMIN." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb" | awk '{print $1}')" = "$WRAPPER_SHA"
|
||||||
|
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/cloudsql_memory_tool.py" | awk '{print $1}')" = "$TOOL_SHA"
|
||||||
|
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_runtime_permissions.py" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
|
||||||
|
test "$(sudo nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/verify_gcp_leoclean_service_environment.py" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
|
||||||
}
|
}
|
||||||
REMOTE
|
REMOTE
|
||||||
)
|
)
|
||||||
|
|
||||||
printf -v verify_context \
|
printf -v verify_context \
|
||||||
'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\n' \
|
'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nSOURCE_REVISION=%q\n' \
|
||||||
"$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \
|
"$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \
|
||||||
"$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \
|
"$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \
|
||||||
"$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA"
|
"$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" "$SOURCE_REVISION"
|
||||||
|
|
||||||
verify_body=$(cat <<'REMOTE'
|
verify_body=$(cat <<'REMOTE'
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
verification_tmp="$(mktemp -d /tmp/teleo-gcp-runtime-verify.XXXXXX)"
|
verification_tmp="$(mktemp -d /tmp/teleo-gcp-runtime-verify.XXXXXX)"
|
||||||
|
sudo chown teleo:teleo "$verification_tmp"
|
||||||
|
sudo chmod 0700 "$verification_tmp"
|
||||||
cleanup_verification() { sudo rm -rf -- "$verification_tmp"; }
|
cleanup_verification() { sudo rm -rf -- "$verification_tmp"; }
|
||||||
trap cleanup_verification EXIT
|
trap cleanup_verification EXIT
|
||||||
assert_deployed_files
|
assert_deployed_files
|
||||||
assert_service_running
|
assert_service_running
|
||||||
assert_runtime_environment
|
assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"
|
||||||
run_scoped_status verify "$REMOTE_PROFILE_BIN/cloudsql_memory_tool.py" "$verification_tmp/gcloud"
|
run_service_wrapper_status verify
|
||||||
|
assert_administrator_secret_denied
|
||||||
|
run_permission_verifier verify "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_runtime_permissions.py" "$verification_tmp/permission-receipt.json"
|
||||||
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
|
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
|
||||||
REMOTE
|
REMOTE
|
||||||
)
|
)
|
||||||
verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}"
|
verify_command="${verify_context}${remote_helpers}"$'\n'"${verify_body}"
|
||||||
|
|
||||||
if $DRY_RUN; then
|
if $DRY_RUN; then
|
||||||
echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE restart=$RESTART"
|
echo "DRY RUN: project=$PROJECT zone=$ZONE instance=$INSTANCE service=$SERVICE preflight_only=$PREFLIGHT_ONLY restart=$RESTART"
|
||||||
echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA dropin_sha256=$DROPIN_SHA"
|
echo "DRY RUN: revision=$SOURCE_REVISION wrapper_sha256=$WRAPPER_SHA tool_sha256=$TOOL_SHA permission_verifier_sha256=$PERMISSION_VERIFIER_SHA service_env_verifier_sha256=$SERVICE_ENV_VERIFIER_SHA dropin_sha256=$DROPIN_SHA"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
@ -236,39 +548,88 @@ if $VERIFY_ONLY; then
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! $RESTART; then
|
if ! $RESTART && ! $PREFLIGHT_ONLY; then
|
||||||
echo "Refusing to install runtime files without --restart; use --dry-run or --verify-only for read-only work." >&2
|
echo "Refusing to install runtime files without --restart; use --dry-run, --preflight-only, or --verify-only for read-only work." >&2
|
||||||
exit 64
|
exit 64
|
||||||
fi
|
fi
|
||||||
|
|
||||||
REMOTE_TMP="/tmp/teleo-gcp-leoclean-runtime-$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
|
||||||
printf -v sync_context \
|
printf -v sync_context \
|
||||||
'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_PROFILE_BIN=%q\nREMOTE_DROPIN_DIR=%q\nREMOTE_CLOUDSDK_CONFIG=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nDROPIN_SHA=%q\nREMOTE_TMP=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\n' \
|
'PROJECT=%q\nSERVICE=%q\nEXPECTED_VM_SERVICE_ACCOUNT=%q\nREMOTE_RUNTIME_BIN=%q\nSERVICE_PROFILE_BIN=%q\nSERVICE_PROFILE_ROOT=%q\nSERVICE_HERMES_ROOT=%q\nREMOTE_DROPIN_DIR=%q\nDROPIN_NAME=%q\nWRAPPER_SHA=%q\nTOOL_SHA=%q\nPERMISSION_VERIFIER_SHA=%q\nSERVICE_ENV_VERIFIER_SHA=%q\nDROPIN_SHA=%q\nWRAPPER_REL=%q\nTOOL_REL=%q\nPERMISSION_VERIFIER_REL=%q\nSERVICE_ENV_VERIFIER_REL=%q\nDROPIN_REL=%q\nSOURCE_REVISION=%q\nPREFLIGHT_ONLY=%q\n' \
|
||||||
"$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_PROFILE_BIN" \
|
"$PROJECT" "$SERVICE" "$EXPECTED_VM_SERVICE_ACCOUNT" "$REMOTE_RUNTIME_BIN" "$SERVICE_PROFILE_BIN" "$SERVICE_PROFILE_ROOT" "$SERVICE_HERMES_ROOT" \
|
||||||
"$REMOTE_DROPIN_DIR" "$REMOTE_CLOUDSDK_CONFIG" "$DROPIN_NAME" \
|
"$REMOTE_DROPIN_DIR" "$DROPIN_NAME" \
|
||||||
"$WRAPPER_SHA" "$TOOL_SHA" "$DROPIN_SHA" "$REMOTE_TMP" \
|
"$WRAPPER_SHA" "$TOOL_SHA" "$PERMISSION_VERIFIER_SHA" "$SERVICE_ENV_VERIFIER_SHA" "$DROPIN_SHA" \
|
||||||
"$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" "$SOURCE_REVISION"
|
"$WRAPPER_REL" "$TOOL_REL" "$PERMISSION_VERIFIER_REL" "$SERVICE_ENV_VERIFIER_REL" "$DROPIN_REL" "$SOURCE_REVISION" "$PREFLIGHT_ONLY"
|
||||||
|
|
||||||
sync_body=$(cat <<'REMOTE'
|
sync_body=$(cat <<'REMOTE'
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
remote_tmp="$REMOTE_TMP"
|
remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)"
|
||||||
profile_bin="$REMOTE_PROFILE_BIN"
|
runtime_bin="$REMOTE_RUNTIME_BIN"
|
||||||
|
runtime_parent="$(dirname "$runtime_bin")"
|
||||||
|
runtime_grandparent="$(dirname "$runtime_parent")"
|
||||||
|
runtime_anchor="$(dirname "$runtime_grandparent")"
|
||||||
dropin_dir="$REMOTE_DROPIN_DIR"
|
dropin_dir="$REMOTE_DROPIN_DIR"
|
||||||
|
dropin_parent="$(dirname "$dropin_dir")"
|
||||||
|
payload_dir="$remote_tmp/payload"
|
||||||
backup_dir="$remote_tmp/backup"
|
backup_dir="$remote_tmp/backup"
|
||||||
|
work_dir="$remote_tmp/work"
|
||||||
|
wrapper_stage="$runtime_bin/.teleo-kb.livingip-new.$$"
|
||||||
|
tool_stage="$runtime_bin/.cloudsql-memory-tool.livingip-new.$$"
|
||||||
|
permission_verifier_stage="$runtime_bin/.runtime-permission-verifier.livingip-new.$$"
|
||||||
|
service_env_verifier_stage="$runtime_bin/.service-environment-verifier.livingip-new.$$"
|
||||||
|
revision_stage="$runtime_bin/.livingip-runtime-revision.livingip-new.$$"
|
||||||
dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$"
|
dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$"
|
||||||
mutation_started=false
|
mutation_started=false
|
||||||
|
|
||||||
|
assert_existing_root_directory() {
|
||||||
|
local target="$1"
|
||||||
|
if sudo test -e "$target" || sudo test -L "$target"; then
|
||||||
|
sudo test -d "$target"
|
||||||
|
sudo test ! -L "$target"
|
||||||
|
test "$(sudo stat -c '%U:%G' "$target")" = "root:root"
|
||||||
|
sudo -n -u teleo test ! -w "$target"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_existing_root_file() {
|
||||||
|
local target="$1"
|
||||||
|
if sudo test -e "$target" || sudo test -L "$target"; then
|
||||||
|
sudo test -f "$target"
|
||||||
|
sudo test ! -L "$target"
|
||||||
|
test "$(sudo stat -c '%U:%G' "$target")" = "root:root"
|
||||||
|
sudo -n -u teleo test ! -w "$target"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
backup_path() {
|
backup_path() {
|
||||||
local target="$1" name="$2"
|
local target="$1" name="$2"
|
||||||
if sudo test -e "$target"; then
|
if sudo test -e "$target"; then
|
||||||
sudo cp -a -- "$target" "$backup_dir/$name"
|
sudo cp -a -- "$target" "$backup_dir/$name"
|
||||||
touch "$backup_dir/$name.present"
|
sudo touch "$backup_dir/$name.present"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
backup_directory_state() {
|
||||||
|
local target="$1" name="$2"
|
||||||
|
if sudo test -d "$target"; then
|
||||||
|
sudo stat -c '%u %g %a' "$target" | sudo tee "$backup_dir/$name.metadata" >/dev/null
|
||||||
|
sudo touch "$backup_dir/$name.present"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
restore_directory_state() {
|
||||||
|
local target="$1" name="$2" uid gid mode
|
||||||
|
if sudo test -f "$backup_dir/$name.present"; then
|
||||||
|
read -r uid gid mode <<<"$(sudo cat "$backup_dir/$name.metadata")"
|
||||||
|
sudo chown "$uid:$gid" "$target"
|
||||||
|
sudo chmod "$mode" "$target"
|
||||||
|
elif sudo test -d "$target"; then
|
||||||
|
sudo rmdir -- "$target"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
restore_path() {
|
restore_path() {
|
||||||
local target="$1" name="$2" restore_rc=0 restore_tmp
|
local target="$1" name="$2" restore_rc=0 restore_tmp
|
||||||
if [ -f "$backup_dir/$name.present" ]; then
|
if sudo test -f "$backup_dir/$name.present"; then
|
||||||
restore_tmp="${target}.livingip-rollback.$$"
|
restore_tmp="${target}.livingip-rollback.$$"
|
||||||
sudo rm -f -- "$restore_tmp" || restore_rc=1
|
sudo rm -f -- "$restore_tmp" || restore_rc=1
|
||||||
if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then
|
if sudo cp -a -- "$backup_dir/$name" "$restore_tmp"; then
|
||||||
|
|
@ -283,15 +644,65 @@ restore_path() {
|
||||||
return "$restore_rc"
|
return "$restore_rc"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
validate_restored_path() {
|
||||||
|
local target="$1" name="$2"
|
||||||
|
if sudo test -f "$backup_dir/$name.present"; then
|
||||||
|
sudo test -f "$target"
|
||||||
|
sudo test ! -L "$target"
|
||||||
|
sudo cmp -s -- "$backup_dir/$name" "$target"
|
||||||
|
test "$(sudo stat -c '%u:%g:%a' "$target")" = \
|
||||||
|
"$(sudo stat -c '%u:%g:%a' "$backup_dir/$name")"
|
||||||
|
else
|
||||||
|
sudo test ! -e "$target"
|
||||||
|
sudo test ! -L "$target"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_restored_directory() {
|
||||||
|
local target="$1" name="$2" expected_state
|
||||||
|
if sudo test -f "$backup_dir/$name.present"; then
|
||||||
|
expected_state="$(sudo cat "$backup_dir/$name.metadata" | tr ' ' ':')"
|
||||||
|
sudo test -d "$target"
|
||||||
|
sudo test ! -L "$target"
|
||||||
|
test "$(sudo stat -c '%u:%g:%a' "$target")" = "$expected_state"
|
||||||
|
else
|
||||||
|
sudo test ! -e "$target"
|
||||||
|
sudo test ! -L "$target"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
rollback_runtime() {
|
rollback_runtime() {
|
||||||
local rollback_rc=0
|
local rollback_rc=0
|
||||||
echo "Deployment failed; restoring the complete previous runtime set." >&2
|
echo "Deployment failed; restoring the complete previous runtime set." >&2
|
||||||
sudo systemctl stop "$SERVICE" || rollback_rc=1
|
sudo systemctl stop "$SERVICE" || rollback_rc=1
|
||||||
restore_path "$profile_bin/teleo-kb" wrapper || rollback_rc=1
|
restore_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1
|
||||||
restore_path "$profile_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
|
restore_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
|
||||||
|
restore_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1
|
||||||
|
restore_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1
|
||||||
restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1
|
restore_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1
|
||||||
restore_path "$profile_bin/.livingip-runtime-revision" revision || rollback_rc=1
|
restore_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1
|
||||||
sudo systemctl daemon-reload || rollback_rc=1
|
restore_directory_state "$runtime_bin" runtime_bin || rollback_rc=1
|
||||||
|
restore_directory_state "$runtime_parent" runtime_parent || rollback_rc=1
|
||||||
|
restore_directory_state "$runtime_grandparent" runtime_grandparent || rollback_rc=1
|
||||||
|
restore_directory_state "$dropin_dir" dropin_dir || rollback_rc=1
|
||||||
|
if [ "$rollback_rc" -eq 0 ]; then
|
||||||
|
validate_restored_path "$runtime_bin/teleo-kb" wrapper || rollback_rc=1
|
||||||
|
validate_restored_path "$runtime_bin/cloudsql_memory_tool.py" tool || rollback_rc=1
|
||||||
|
validate_restored_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier || rollback_rc=1
|
||||||
|
validate_restored_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier || rollback_rc=1
|
||||||
|
validate_restored_path "$dropin_dir/$DROPIN_NAME" dropin || rollback_rc=1
|
||||||
|
validate_restored_path "$runtime_bin/.livingip-runtime-revision" revision || rollback_rc=1
|
||||||
|
validate_restored_directory "$runtime_grandparent" runtime_grandparent || rollback_rc=1
|
||||||
|
validate_restored_directory "$runtime_parent" runtime_parent || rollback_rc=1
|
||||||
|
validate_restored_directory "$runtime_bin" runtime_bin || rollback_rc=1
|
||||||
|
validate_restored_directory "$dropin_dir" dropin_dir || rollback_rc=1
|
||||||
|
fi
|
||||||
|
if [ "$rollback_rc" -eq 0 ]; then
|
||||||
|
sudo systemctl daemon-reload || rollback_rc=1
|
||||||
|
fi
|
||||||
|
if [ "$rollback_rc" -eq 0 ]; then
|
||||||
|
assert_unit_configuration false || rollback_rc=1
|
||||||
|
fi
|
||||||
if [ "$rollback_rc" -eq 0 ]; then
|
if [ "$rollback_rc" -eq 0 ]; then
|
||||||
sudo systemctl start "$SERVICE" || rollback_rc=1
|
sudo systemctl start "$SERVICE" || rollback_rc=1
|
||||||
assert_service_running || rollback_rc=1
|
assert_service_running || rollback_rc=1
|
||||||
|
|
@ -312,7 +723,13 @@ on_exit() {
|
||||||
rc=1
|
rc=1
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
sudo rm -f -- "$dropin_stage" || true
|
sudo rm -f -- \
|
||||||
|
"$wrapper_stage" \
|
||||||
|
"$tool_stage" \
|
||||||
|
"$permission_verifier_stage" \
|
||||||
|
"$service_env_verifier_stage" \
|
||||||
|
"$revision_stage" \
|
||||||
|
"$dropin_stage" || true
|
||||||
sudo rm -rf -- "$remote_tmp" || true
|
sudo rm -rf -- "$remote_tmp" || true
|
||||||
exit "$rc"
|
exit "$rc"
|
||||||
}
|
}
|
||||||
|
|
@ -321,36 +738,117 @@ trap 'on_exit $?' EXIT
|
||||||
trap 'exit 130' INT
|
trap 'exit 130' INT
|
||||||
trap 'exit 143' TERM
|
trap 'exit 143' TERM
|
||||||
|
|
||||||
sudo rm -rf -- "$remote_tmp"
|
sudo chmod 0711 "$remote_tmp"
|
||||||
mkdir -p "$backup_dir"
|
sudo install -d -o root -g root -m 0755 "$payload_dir"
|
||||||
tar -C "$remote_tmp" -xf -
|
sudo install -d -o root -g root -m 0700 "$backup_dir"
|
||||||
|
sudo install -d -o teleo -g teleo -m 0700 "$work_dir"
|
||||||
|
sudo tar --no-same-owner --no-same-permissions --no-xattrs --no-acls --no-selinux -C "$payload_dir" -xf -
|
||||||
|
if sudo find "$payload_dir" -type l -print -quit | grep -q .; then
|
||||||
|
echo "Staged runtime payload contains a symlink." >&2
|
||||||
|
exit 65
|
||||||
|
fi
|
||||||
|
if sudo find "$payload_dir" ! -type d ! -type f -print -quit | grep -q .; then
|
||||||
|
echo "Staged runtime payload contains a non-regular entry." >&2
|
||||||
|
exit 65
|
||||||
|
fi
|
||||||
|
sudo chown -R root:root "$payload_dir"
|
||||||
|
sudo find "$payload_dir" -type d -exec chmod 0555 {} +
|
||||||
|
sudo find "$payload_dir" -type f -exec chmod 0444 {} +
|
||||||
|
sudo chmod 0555 \
|
||||||
|
"$payload_dir/$WRAPPER_REL" \
|
||||||
|
"$payload_dir/$TOOL_REL" \
|
||||||
|
"$payload_dir/$PERMISSION_VERIFIER_REL" \
|
||||||
|
"$payload_dir/$SERVICE_ENV_VERIFIER_REL"
|
||||||
|
test "$(sudo sha256sum "$payload_dir/$WRAPPER_REL" | awk '{print $1}')" = "$WRAPPER_SHA"
|
||||||
|
test "$(sudo sha256sum "$payload_dir/$TOOL_REL" | awk '{print $1}')" = "$TOOL_SHA"
|
||||||
|
test "$(sudo sha256sum "$payload_dir/$PERMISSION_VERIFIER_REL" | awk '{print $1}')" = "$PERMISSION_VERIFIER_SHA"
|
||||||
|
test "$(sudo sha256sum "$payload_dir/$SERVICE_ENV_VERIFIER_REL" | awk '{print $1}')" = "$SERVICE_ENV_VERIFIER_SHA"
|
||||||
|
test "$(sudo sha256sum "$payload_dir/$DROPIN_REL" | awk '{print $1}')" = "$DROPIN_SHA"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$remote_tmp")" = "root:root:711"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$payload_dir")" = "root:root:555"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$backup_dir")" = "root:root:700"
|
||||||
|
test "$(sudo stat -c '%U:%G:%a' "$work_dir")" = "teleo:teleo:700"
|
||||||
|
sudo -n -u teleo test ! -w "$remote_tmp"
|
||||||
|
sudo -n -u teleo test ! -w "$payload_dir"
|
||||||
|
sudo -n -u teleo test ! -w "$backup_dir"
|
||||||
|
sudo test -d "$runtime_anchor"
|
||||||
|
sudo test -d "$dropin_parent"
|
||||||
|
sudo test ! -L "$runtime_anchor"
|
||||||
|
sudo test ! -L "$dropin_parent"
|
||||||
|
test "$(sudo stat -c '%U:%G' "$runtime_anchor")" = "root:root"
|
||||||
|
test "$(sudo stat -c '%U:%G' "$dropin_parent")" = "root:root"
|
||||||
|
sudo -n -u teleo test ! -w "$runtime_anchor"
|
||||||
|
sudo -n -u teleo test ! -w "$dropin_parent"
|
||||||
|
for existing_directory in \
|
||||||
|
"$runtime_grandparent" \
|
||||||
|
"$runtime_parent" \
|
||||||
|
"$runtime_bin" \
|
||||||
|
"$dropin_dir"; do
|
||||||
|
assert_existing_root_directory "$existing_directory"
|
||||||
|
done
|
||||||
|
for existing_file in \
|
||||||
|
"$runtime_bin/teleo-kb" \
|
||||||
|
"$runtime_bin/cloudsql_memory_tool.py" \
|
||||||
|
"$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" \
|
||||||
|
"$runtime_bin/verify_gcp_leoclean_service_environment.py" \
|
||||||
|
"$runtime_bin/.livingip-runtime-revision" \
|
||||||
|
"$dropin_dir/$DROPIN_NAME"; do
|
||||||
|
assert_existing_root_file "$existing_file"
|
||||||
|
done
|
||||||
|
|
||||||
assert_service_running
|
assert_service_running
|
||||||
run_scoped_status pre "$remote_tmp/$TOOL_REL" "$remote_tmp/gcloud-pre"
|
preflight_service_before="$(service_fingerprint)"
|
||||||
|
assert_runtime_environment "$payload_dir/$SERVICE_ENV_VERIFIER_REL" false
|
||||||
|
run_scoped_status pre "$payload_dir/$TOOL_REL"
|
||||||
|
assert_administrator_secret_denied
|
||||||
|
run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL" "$work_dir/permission-pre/receipt.json"
|
||||||
|
assert_service_fingerprint "$preflight_service_before"
|
||||||
|
if [ "$PREFLIGHT_ONLY" = true ]; then
|
||||||
|
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
backup_path "$profile_bin/teleo-kb" wrapper
|
backup_path "$runtime_bin/teleo-kb" wrapper
|
||||||
backup_path "$profile_bin/cloudsql_memory_tool.py" tool
|
backup_path "$runtime_bin/cloudsql_memory_tool.py" tool
|
||||||
|
backup_path "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" verifier
|
||||||
|
backup_path "$runtime_bin/verify_gcp_leoclean_service_environment.py" service_env_verifier
|
||||||
backup_path "$dropin_dir/$DROPIN_NAME" dropin
|
backup_path "$dropin_dir/$DROPIN_NAME" dropin
|
||||||
backup_path "$profile_bin/.livingip-runtime-revision" revision
|
backup_path "$runtime_bin/.livingip-runtime-revision" revision
|
||||||
|
backup_directory_state "$runtime_parent" runtime_parent
|
||||||
|
backup_directory_state "$runtime_grandparent" runtime_grandparent
|
||||||
|
backup_directory_state "$runtime_bin" runtime_bin
|
||||||
|
backup_directory_state "$dropin_dir" dropin_dir
|
||||||
|
|
||||||
mutation_started=true
|
mutation_started=true
|
||||||
sudo systemctl stop "$SERVICE"
|
sudo systemctl stop "$SERVICE"
|
||||||
sudo install -d -o teleo -g teleo -m 0755 "$profile_bin"
|
sudo install -d -o root -g root -m 0755 "$runtime_parent"
|
||||||
sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$WRAPPER_REL" "$profile_bin/teleo-kb"
|
sudo install -d -o root -g root -m 0755 "$runtime_bin"
|
||||||
sudo install -o teleo -g teleo -m 0755 "$remote_tmp/$TOOL_REL" "$profile_bin/cloudsql_memory_tool.py"
|
sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage"
|
||||||
|
sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage"
|
||||||
|
sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" "$permission_verifier_stage"
|
||||||
|
sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" "$service_env_verifier_stage"
|
||||||
|
sudo mv -f -- "$wrapper_stage" "$runtime_bin/teleo-kb"
|
||||||
|
sudo mv -f -- "$tool_stage" "$runtime_bin/cloudsql_memory_tool.py"
|
||||||
|
sudo mv -f -- "$permission_verifier_stage" "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py"
|
||||||
|
sudo mv -f -- "$service_env_verifier_stage" "$runtime_bin/verify_gcp_leoclean_service_environment.py"
|
||||||
sudo install -d -o root -g root -m 0755 "$dropin_dir"
|
sudo install -d -o root -g root -m 0755 "$dropin_dir"
|
||||||
sudo install -o root -g root -m 0644 "$remote_tmp/$DROPIN_REL" "$dropin_stage"
|
sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL" "$dropin_stage"
|
||||||
sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME"
|
sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME"
|
||||||
printf '%s\n' "$SOURCE_REVISION" | sudo tee "$profile_bin/.livingip-runtime-revision" >/dev/null
|
printf '%s\n' "$SOURCE_REVISION" | sudo tee "$revision_stage" >/dev/null
|
||||||
sudo chown teleo:teleo "$profile_bin/.livingip-runtime-revision"
|
sudo chown root:root "$revision_stage"
|
||||||
sudo chmod 0644 "$profile_bin/.livingip-runtime-revision"
|
sudo chmod 0644 "$revision_stage"
|
||||||
|
sudo mv -f -- "$revision_stage" "$runtime_bin/.livingip-runtime-revision"
|
||||||
|
assert_installed_files
|
||||||
sudo systemctl daemon-reload
|
sudo systemctl daemon-reload
|
||||||
|
assert_unit_configuration
|
||||||
sudo systemctl start "$SERVICE"
|
sudo systemctl start "$SERVICE"
|
||||||
|
|
||||||
assert_deployed_files
|
assert_deployed_files
|
||||||
assert_service_running
|
assert_service_running
|
||||||
assert_runtime_environment
|
assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py"
|
||||||
run_scoped_status post "$profile_bin/cloudsql_memory_tool.py" "$remote_tmp/gcloud-post"
|
run_service_wrapper_status post
|
||||||
|
assert_administrator_secret_denied
|
||||||
|
run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py" "$work_dir/permission-post/receipt.json"
|
||||||
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
|
systemctl show "$SERVICE" -p ActiveState -p SubState -p NRestarts -p MainPID -p User --no-pager
|
||||||
mutation_started=false
|
mutation_started=false
|
||||||
REMOTE
|
REMOTE
|
||||||
|
|
@ -358,4 +856,8 @@ REMOTE
|
||||||
sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}"
|
sync_command="${sync_context}${remote_helpers}"$'\n'"${sync_body}"
|
||||||
|
|
||||||
COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \
|
COPYFILE_DISABLE=1 tar --format=ustar -C "$REPO_ROOT" -cf - \
|
||||||
"$WRAPPER_REL" "$TOOL_REL" "$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command"
|
"$WRAPPER_REL" \
|
||||||
|
"$TOOL_REL" \
|
||||||
|
"$PERMISSION_VERIFIER_REL" \
|
||||||
|
"$SERVICE_ENV_VERIFIER_REL" \
|
||||||
|
"$DROPIN_REL" | "${GCP_SSH[@]}" --command="$sync_command"
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,9 @@
|
||||||
|
|
||||||
## Scope
|
## Scope
|
||||||
|
|
||||||
This runbook reconciles PR `#145` with the non-production GCP parallel Leo
|
This runbook reconciles PR `#148`, the least-privilege follow-up to PR `#145`,
|
||||||
service and removes its dependency on the PostgreSQL administrator credential.
|
with the non-production GCP parallel Leo service and removes its dependency on
|
||||||
|
the PostgreSQL administrator credential.
|
||||||
It does not promote GCP, change the Telegram destination, copy newer VPS data,
|
It does not promote GCP, change the Telegram destination, copy newer VPS data,
|
||||||
or make `teleo_canonical` production-authoritative.
|
or make `teleo_canonical` production-authoritative.
|
||||||
|
|
||||||
|
|
@ -31,7 +32,8 @@ IAM or networking.
|
||||||
commit.
|
commit.
|
||||||
2. `TELEO_KB_MODE=cloudsql`; missing tools or credentials fail closed.
|
2. `TELEO_KB_MODE=cloudsql`; missing tools or credentials fail closed.
|
||||||
3. Canonical zero-hit searches do not consult `teleo_restore` unless an
|
3. Canonical zero-hit searches do not consult `teleo_restore` unless an
|
||||||
operator explicitly opts in.
|
operator explicitly opts in. The service is pinned to database
|
||||||
|
`teleo_canonical` with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`.
|
||||||
4. Leo can read the named canonical tables and stage only a
|
4. Leo can read the named canonical tables and stage only a
|
||||||
`pending_review` proposal through `kb_stage.stage_leoclean_proposal(...)`.
|
`pending_review` proposal through `kb_stage.stage_leoclean_proposal(...)`.
|
||||||
5. Leo cannot directly insert, update, or delete `public.*` or
|
5. Leo cannot directly insert, update, or delete `public.*` or
|
||||||
|
|
@ -55,7 +57,7 @@ required runtime secret has an equivalent secret-level binding.
|
||||||
|
|
||||||
### 2. Merge reviewed repository code
|
### 2. Merge reviewed repository code
|
||||||
|
|
||||||
Run CI on the completed PR `#145` repair before deployment. The deployment
|
Run CI on the completed PR `#148` repair before deployment. The deployment
|
||||||
source must be the resulting merged commit, not a working tree or unmerged
|
source must be the resulting merged commit, not a working tree or unmerged
|
||||||
branch.
|
branch.
|
||||||
|
|
||||||
|
|
@ -73,6 +75,9 @@ private VM path. Supply the new runtime password through
|
||||||
`TELEO_LEOCLEAN_DB_PASSWORD`; the SQL file does not contain it. The migration:
|
`TELEO_LEOCLEAN_DB_PASSWORD`; the SQL file does not contain it. The migration:
|
||||||
|
|
||||||
- creates or rotates `leoclean_kb_runtime`;
|
- creates or rotates `leoclean_kb_runtime`;
|
||||||
|
- keeps `leoclean_kb_runtime` `NOLOGIN` while both the `teleo_canonical` grant
|
||||||
|
phase and the legacy `teleo_kb` denial-verification phase run, and enables
|
||||||
|
`LOGIN` only in the final transaction after both phases pass;
|
||||||
- creates a dedicated `NOLOGIN` function owner with no role memberships;
|
- creates a dedicated `NOLOGIN` function owner with no role memberships;
|
||||||
- removes stale table, column, sequence, function, and role-membership grants;
|
- removes stale table, column, sequence, function, and role-membership grants;
|
||||||
- grants exact canonical reads;
|
- grants exact canonical reads;
|
||||||
|
|
@ -94,26 +99,116 @@ by a `pg_catalog, pg_temp` search path, schema-qualified relations, a fixed
|
||||||
The administrator password is used only for this bounded bootstrap. Retain no
|
The administrator password is used only for this bounded bootstrap. Retain no
|
||||||
password output.
|
password output.
|
||||||
|
|
||||||
|
`teleo_kb` can still be connectable through its existing `PUBLIC CONNECT`
|
||||||
|
grant. That database-wide default is not treated as runtime authority. The
|
||||||
|
migration denies the scoped role access to the `teleo_restore` schema, the
|
||||||
|
runtime client is pinned to `teleo_canonical`, and audit fallback remains
|
||||||
|
disabled with `TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0`.
|
||||||
|
|
||||||
### 5. Preflight and deploy the merged runtime
|
### 5. Preflight and deploy the merged runtime
|
||||||
|
|
||||||
From the exact merged checkout, run:
|
From the exact merged checkout, run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
deploy/sync-gcp-leoclean-runtime.sh --dry-run
|
deploy/sync-gcp-leoclean-runtime.sh --dry-run
|
||||||
|
deploy/sync-gcp-leoclean-runtime.sh --preflight-only
|
||||||
deploy/sync-gcp-leoclean-runtime.sh --restart
|
deploy/sync-gcp-leoclean-runtime.sh --restart
|
||||||
```
|
```
|
||||||
|
|
||||||
The deploy script runs a scoped, redacted database status preflight before it
|
`--preflight-only` extracts the candidate helper and permission verifier into a
|
||||||
installs files or restarts the service. It refuses dirty or unmerged runtime
|
root-created, root-owned, non-writable VM payload directory, verifies the actual
|
||||||
files and refuses a live install without `--restart`. After preflight it stops
|
service `MainPID` environment, runs the candidate private Cloud SQL status path,
|
||||||
the parallel service, atomically replaces the existing Cloud SQL systemd
|
and runs the sanitized positive-and-negative permission verifier. A separate
|
||||||
drop-in, installs the reviewed wrapper and helper, records the Git revision,
|
`teleo`-owned directory holds only sanitized receipts; it never feeds an install
|
||||||
starts the service, and verifies all three file hashes. A failure after mutation
|
or rollback. The preflight does not stop or restart the service and does not
|
||||||
restores the prior wrapper, helper, drop-in, and revision as one set before the
|
install or replace runtime files. It
|
||||||
old service is restarted.
|
captures `ActiveState`, `SubState`, `MainPID`, and `NRestarts` before and after
|
||||||
|
the checks and fails if any value changes. The preflight must pass before
|
||||||
|
`--restart` is attempted.
|
||||||
|
|
||||||
|
The deploy script refuses dirty, symlinked, or unmerged runtime files and
|
||||||
|
refuses a live install without `--restart`. Its payload and rollback backup are
|
||||||
|
root-owned and non-writable by `teleo`; existing runtime intermediates and
|
||||||
|
targets must be real root-owned, non-writable paths. After preflight it stops
|
||||||
|
the parallel service, atomically replaces the existing Cloud SQL systemd drop-in, installs the
|
||||||
|
reviewed wrapper, helper, and permission verifier into the root-controlled
|
||||||
|
`/usr/local/libexec/livingip/leoclean-kb` directory, records the Git revision,
|
||||||
|
and verifies every installed hash, owner, mode, path type, and revision before
|
||||||
|
systemd reloads the unit or starts the service. The wrapper uses fixed
|
||||||
|
`/bin/bash` and `/usr/bin/python3` interpreters, and every directory on the
|
||||||
|
service `PATH` must be root-owned and non-writable by `teleo`. Every ancestor of that
|
||||||
|
directory, the directory itself, the wrapper, helper, verifier, and revision
|
||||||
|
marker must be non-writable by `teleo`. The systemd service receives that
|
||||||
|
directory as a read-only bind at its existing
|
||||||
|
`/home/teleo/.hermes/profiles/leoclean/bin` path; verification compares the
|
||||||
|
source and service-namespace inode and hashes and proves the mount is read-only.
|
||||||
|
To prevent the `teleo` service user from renaming a writable ancestor and
|
||||||
|
recreating that absolute path, the service namespace mounts
|
||||||
|
`/home/teleo/.hermes` read-only, allows writes back only to the profile's
|
||||||
|
`state` and `workspace` subdirectories, and removes `CAP_SYS_ADMIN`. Live
|
||||||
|
verification requires `.hermes` and `bin` to be read-only, both allowed
|
||||||
|
subdirectories to be writable, `/home` to be non-writable by `teleo`, and the
|
||||||
|
effective capability set to exclude `CAP_SYS_ADMIN`.
|
||||||
|
|
||||||
|
Preflight, post-restart, and `--verify-only` checks inspect the environment of the actual
|
||||||
|
systemd `MainPID` through `/proc/<MainPID>/environ`, rather than trusting the
|
||||||
|
configured unit environment alone. A standalone fail-closed verifier requires
|
||||||
|
the exact reviewed private target and runtime role, requires
|
||||||
|
`TELEO_GCP_METADATA_ONLY=1`, and rejects all PostgreSQL, Cloud SDK, proxy, TLS
|
||||||
|
trust, broader Google credential, shell-startup, dynamic-loader, Python-loader,
|
||||||
|
and administrator-secret overrides without printing their values. Effective
|
||||||
|
`EnvironmentFile` use and drop-ins ordered after the reviewed configuration are
|
||||||
|
rejected before restart. This blocks
|
||||||
|
pre-wrapper injection through fields such as `BASH_ENV`, any `LD_*`,
|
||||||
|
`PYTHONPATH`, `PYTHONHTTPSVERIFY`, or `SSLKEYLOGFILE`. The checks also require
|
||||||
|
the reviewed drop-in to appear in `DropInPaths`. The wrapper status probe then
|
||||||
|
imports the validated environment from the actual `MainPID` with `nsenter
|
||||||
|
--env`, enters its mount namespace, drops to `teleo`, and invokes the bound
|
||||||
|
wrapper without reinjecting any database identity setting.
|
||||||
|
|
||||||
|
The service runtime bypasses Cloud SDK configuration and credential caches
|
||||||
|
entirely. The helper obtains the attached service-account email and a bounded
|
||||||
|
access token from the fixed GCE metadata endpoint, requires the exact expected
|
||||||
|
service account, and accesses only the named scoped Secret Manager version over
|
||||||
|
the fixed HTTPS API. Its URL opener disables proxies. The scoped status read
|
||||||
|
and administrator-secret IAM denial are rerun after restart. The denial probe
|
||||||
|
imports the actual systemd `MainPID` environment and mount namespace, drops to
|
||||||
|
the service's `teleo` identity, and must observe the exact
|
||||||
|
`secretmanager.versions.access` permission denial. No token or
|
||||||
|
secret value is retained. The status read executes the bound `teleo-kb` wrapper
|
||||||
|
from inside the service mount namespace, proving the wrapper and its adjacent
|
||||||
|
helper rather than bypassing them with a direct helper invocation. A failure
|
||||||
|
after mutation restores the prior runtime/drop-in directory existence and
|
||||||
|
metadata, wrapper, helper, both verifiers, drop-in, and revision as one set,
|
||||||
|
validates it against the root-only backup, and only then reloads systemd and
|
||||||
|
restarts the old service.
|
||||||
|
|
||||||
### 6. Verify positive and negative behavior
|
### 6. Verify positive and negative behavior
|
||||||
|
|
||||||
|
`--preflight-only`, `--restart`, and `--verify-only` run the verifier as the
|
||||||
|
`teleo` Unix user. To rerun the installed verifier directly on the VM and
|
||||||
|
retain its sanitized JSON receipt, use:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
receipt_dir="$(mktemp -d /tmp/leoclean-permission-receipt.XXXXXX)"
|
||||||
|
sudo chown teleo:teleo "$receipt_dir"
|
||||||
|
sudo chmod 0700 "$receipt_dir"
|
||||||
|
run_id="$(date -u +%Y%m%d%H%M%S)-manual"
|
||||||
|
sudo -n -u teleo env -i HOME=/home/teleo \
|
||||||
|
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||||
|
python3 /usr/local/libexec/livingip/leoclean-kb/verify_gcp_leoclean_runtime_permissions.py \
|
||||||
|
--run-id "$run_id" \
|
||||||
|
--output "$receipt_dir/permission-receipt.json"
|
||||||
|
sudo stat -c '%U:%G:%a' "$receipt_dir/permission-receipt.json"
|
||||||
|
```
|
||||||
|
|
||||||
|
The command must exit zero, print the same canonical JSON that it writes, and
|
||||||
|
leave the receipt as `teleo:teleo:600`. A passing receipt must report
|
||||||
|
`status=pass`, `mode=live_private_gcp_staging`, and both `required_tier` and
|
||||||
|
`current_tier` as `T3_live_readonly`. Those values are a receipt contract, not
|
||||||
|
a claim that this runbook has already been exercised successfully on the live
|
||||||
|
staging VM.
|
||||||
|
|
||||||
Retain a redacted receipt containing:
|
Retain a redacted receipt containing:
|
||||||
|
|
||||||
- merged Git commit and deployed file hashes;
|
- merged Git commit and deployed file hashes;
|
||||||
|
|
@ -122,13 +217,28 @@ Retain a redacted receipt containing:
|
||||||
configuration;
|
configuration;
|
||||||
- `current_database()` and `current_user` showing
|
- `current_database()` and `current_user` showing
|
||||||
`teleo_canonical|leoclean_kb_runtime`;
|
`teleo_canonical|leoclean_kb_runtime`;
|
||||||
|
- private server address `10.61.0.3`, port `5432`, and an active SSL session;
|
||||||
|
- exact safe runtime-role attributes and zero membership edges in
|
||||||
|
`pg_auth_members`;
|
||||||
|
- zero effective database/schema `CREATE`, persistent ownership, table/column
|
||||||
|
DML, sequence privileges, SELECT outside the exact table allowlist, or
|
||||||
|
executable unexpected `SECURITY DEFINER` functions, plus the exact staging
|
||||||
|
function owner/search-path/ACL contract;
|
||||||
- a real canonical status/search receipt;
|
- a real canonical status/search receipt;
|
||||||
- a transaction-rolled-back call to `stage_leoclean_proposal(...)` that returns
|
- a transaction-rolled-back call to `stage_leoclean_proposal(...)` that returns
|
||||||
`pending_review`;
|
`pending_review` for canonical proposer `leo`, with zero matching canary rows
|
||||||
- denied direct insert into `kb_stage.kb_proposals`;
|
both before and after the transaction;
|
||||||
- denied canonical `public.*` write;
|
- denied direct insert, update, and delete on `kb_stage.kb_proposals`;
|
||||||
- denied reviewer/apply security-definer function;
|
- denied insert, update, and delete on canonical `public.claims`;
|
||||||
- denied forged proposer identity and denied `SET ROLE` escalation;
|
- denied reviewer/apply security-definer functions;
|
||||||
|
- exact function-catalog posture: only the five-argument staging function is
|
||||||
|
executable, the forged six-argument overload is absent, and each expected
|
||||||
|
reviewer/apply function exists but is not executable by the runtime role;
|
||||||
|
- unavailable forged-proposer overload and denied `SET ROLE` escalation into
|
||||||
|
the staging owner, reviewer, apply, or administrator roles;
|
||||||
|
- denied access to `teleo_restore` even when `teleo_kb` remains connectable;
|
||||||
|
- readable scoped runtime secret and an IAM permission denial for the
|
||||||
|
administrator secret, with neither secret value retained;
|
||||||
- zero Telegram messages and zero committed canary rows.
|
- zero Telegram messages and zero committed canary rows.
|
||||||
|
|
||||||
### 7. Remove administrator-secret access
|
### 7. Remove administrator-secret access
|
||||||
|
|
|
||||||
|
|
@ -1,20 +1,24 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Cloud SQL memory and KB-proposal bridge for the GCP leoclean profile.
|
"""Cloud SQL memory and KB-proposal bridge for the GCP leoclean profile.
|
||||||
|
|
||||||
Canonical claim/strategy application remains review-gated. Read commands query
|
Canonical claim/strategy application remains review-gated. The production-like
|
||||||
the canonical Teleo KB first, then fall back to the restored `teleo_restore`
|
Leo runtime is pinned to the private canonical Cloud SQL database and never
|
||||||
shadow schema in Cloud SQL. Proposal commands write staged review records into
|
falls back to the restored `teleo_restore` shadow schema. Proposal commands
|
||||||
`kb_stage.kb_proposals`; they do not apply canonical truth directly.
|
write staged review records into `kb_stage.kb_proposals`; they do not apply
|
||||||
|
canonical truth directly.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import base64
|
||||||
|
import binascii
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import urllib.request
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
STOPWORDS = {
|
STOPWORDS = {
|
||||||
|
|
@ -76,6 +80,38 @@ DEFAULT_CLAIM_BASE_URL = "https://leo.livingip.xyz"
|
||||||
RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1"
|
RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1"
|
||||||
RUNTIME_DB_USER = "leoclean_kb_runtime"
|
RUNTIME_DB_USER = "leoclean_kb_runtime"
|
||||||
RUNTIME_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password"
|
RUNTIME_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password"
|
||||||
|
RUNTIME_GCP_PROJECT = "teleo-501523"
|
||||||
|
RUNTIME_CLOUDSQL_HOST = "10.61.0.3"
|
||||||
|
RUNTIME_CLOUDSQL_PORT = "5432"
|
||||||
|
RUNTIME_CLOUDSQL_DB = "teleo_canonical"
|
||||||
|
RUNTIME_SERVICE_ACCOUNT = "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com"
|
||||||
|
GCE_METADATA_EMAIL_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email"
|
||||||
|
GCE_METADATA_TOKEN_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token"
|
||||||
|
RUNTIME_SECRET_ACCESS_URL = (
|
||||||
|
"https://secretmanager.googleapis.com/v1/projects/teleo-501523/secrets/"
|
||||||
|
"gcp-teleo-pgvector-standby-leoclean-kb-runtime-password/versions/latest:access"
|
||||||
|
)
|
||||||
|
METADATA_HEADERS = {"Metadata-Flavor": "Google"}
|
||||||
|
HTTP_TIMEOUT_SECONDS = 3.0
|
||||||
|
MAX_METADATA_EMAIL_BYTES = 256
|
||||||
|
MAX_METADATA_TOKEN_BYTES = 4096
|
||||||
|
MAX_SECRET_RESPONSE_BYTES = 8192
|
||||||
|
MAX_RUNTIME_SECRET_BYTES = 4096
|
||||||
|
RUNTIME_TLS_TRUST_ENVIRONMENT = frozenset(
|
||||||
|
{
|
||||||
|
"CURL_CA_BUNDLE",
|
||||||
|
"PYTHONHTTPSVERIFY",
|
||||||
|
"REQUESTS_CA_BUNDLE",
|
||||||
|
"SSL_CERT_DIR",
|
||||||
|
"SSL_CERT_FILE",
|
||||||
|
"SSLKEYLOGFILE",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
RUNTIME_PSQL_ENV_BASE = {
|
||||||
|
"PATH": "/usr/bin:/bin",
|
||||||
|
"PGSSLMODE": "require",
|
||||||
|
"PGCONNECT_TIMEOUT": "8",
|
||||||
|
}
|
||||||
CLONE_READ_ONLY_PGOPTIONS = "-c default_transaction_read_only=on"
|
CLONE_READ_ONLY_PGOPTIONS = "-c default_transaction_read_only=on"
|
||||||
RECEIPTED_READ_COMMANDS = frozenset(
|
RECEIPTED_READ_COMMANDS = frozenset(
|
||||||
{
|
{
|
||||||
|
|
@ -93,6 +129,18 @@ RECEIPTED_READ_COMMANDS = frozenset(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _RejectRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, *_args: Any, **_kwargs: Any) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
RUNTIME_PROXY_HANDLER = urllib.request.ProxyHandler({})
|
||||||
|
RUNTIME_HTTP_OPENER = urllib.request.build_opener(
|
||||||
|
RUNTIME_PROXY_HANDLER,
|
||||||
|
_RejectRedirectHandler(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
parser.add_argument("--host", default=os.environ.get("TELEO_CLOUDSQL_HOST", "10.61.0.3"))
|
parser.add_argument("--host", default=os.environ.get("TELEO_CLOUDSQL_HOST", "10.61.0.3"))
|
||||||
|
|
@ -246,8 +294,47 @@ def validate_runtime_connection(args: argparse.Namespace) -> None:
|
||||||
raise SystemExit(f"Leo runtime requires database user {RUNTIME_DB_USER}.")
|
raise SystemExit(f"Leo runtime requires database user {RUNTIME_DB_USER}.")
|
||||||
if password_secret != RUNTIME_PASSWORD_SECRET:
|
if password_secret != RUNTIME_PASSWORD_SECRET:
|
||||||
raise SystemExit(f"Leo runtime requires scoped password secret {RUNTIME_PASSWORD_SECRET}.")
|
raise SystemExit(f"Leo runtime requires scoped password secret {RUNTIME_PASSWORD_SECRET}.")
|
||||||
if "PGPASSWORD" in os.environ:
|
expected_fields = {
|
||||||
raise SystemExit("Refusing inherited PGPASSWORD in Leo runtime mode.")
|
"project": RUNTIME_GCP_PROJECT,
|
||||||
|
"host": RUNTIME_CLOUDSQL_HOST,
|
||||||
|
"port": RUNTIME_CLOUDSQL_PORT,
|
||||||
|
"db": RUNTIME_CLOUDSQL_DB,
|
||||||
|
"canonical_db": RUNTIME_CLOUDSQL_DB,
|
||||||
|
}
|
||||||
|
for field, expected in expected_fields.items():
|
||||||
|
actual = str(getattr(args, field, "") or "")
|
||||||
|
if actual != expected:
|
||||||
|
raise SystemExit(f"Leo runtime requires {field}={expected}.")
|
||||||
|
if os.environ.get("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0") != "0":
|
||||||
|
raise SystemExit("Leo runtime requires TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0.")
|
||||||
|
if os.environ.get("TELEO_GCP_METADATA_ONLY") != "1":
|
||||||
|
raise SystemExit("Leo runtime requires TELEO_GCP_METADATA_ONLY=1.")
|
||||||
|
forbidden_environment = any(
|
||||||
|
normalized_key.startswith("PG")
|
||||||
|
or normalized_key.startswith("CLOUDSDK_")
|
||||||
|
or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS"
|
||||||
|
or normalized_key.startswith("LD_")
|
||||||
|
or normalized_key.startswith("BASH_FUNC_")
|
||||||
|
or (normalized_key.startswith("PYTHON") and normalized_key != "PYTHONUNBUFFERED")
|
||||||
|
or normalized_key in {
|
||||||
|
"BASH_ENV",
|
||||||
|
"BASHOPTS",
|
||||||
|
"CDPATH",
|
||||||
|
"ENV",
|
||||||
|
"GLOBIGNORE",
|
||||||
|
"IFS",
|
||||||
|
"PS4",
|
||||||
|
"SHELLOPTS",
|
||||||
|
}
|
||||||
|
or normalized_key == "PROXY"
|
||||||
|
or normalized_key.endswith("_PROXY")
|
||||||
|
or normalized_key in RUNTIME_TLS_TRUST_ENVIRONMENT
|
||||||
|
for normalized_key in (key.upper() for key in os.environ)
|
||||||
|
)
|
||||||
|
if forbidden_environment:
|
||||||
|
raise SystemExit(
|
||||||
|
"Refusing inherited credential, proxy, TLS, or PostgreSQL environment in Leo runtime mode."
|
||||||
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
if mode != "clone-readonly":
|
if mode != "clone-readonly":
|
||||||
|
|
@ -261,8 +348,8 @@ def validate_runtime_connection(args: argparse.Namespace) -> None:
|
||||||
raise SystemExit("Clone credential mode permits read commands only.")
|
raise SystemExit("Clone credential mode permits read commands only.")
|
||||||
if pgoptions != CLONE_READ_ONLY_PGOPTIONS:
|
if pgoptions != CLONE_READ_ONLY_PGOPTIONS:
|
||||||
raise SystemExit("Clone credential mode requires server-enforced default_transaction_read_only=on.")
|
raise SystemExit("Clone credential mode requires server-enforced default_transaction_read_only=on.")
|
||||||
if not user or (not password_secret and not os.environ.get("PGPASSWORD")):
|
if not user or not os.environ.get("PGPASSWORD"):
|
||||||
raise SystemExit("Clone credential mode requires an explicit user and credential source.")
|
raise SystemExit("Clone credential mode requires an explicit user and inherited test credential.")
|
||||||
|
|
||||||
|
|
||||||
def terms_for(query: str) -> list[str]:
|
def terms_for(query: str) -> list[str]:
|
||||||
|
|
@ -305,48 +392,138 @@ def sql_array(values: list[str], cast: str = "text") -> str:
|
||||||
return "array[" + ",".join(sql_literal(value) for value in values) + f"]::{cast}[]"
|
return "array[" + ",".join(sql_literal(value) for value in values) + f"]::{cast}[]"
|
||||||
|
|
||||||
|
|
||||||
|
def _read_http_response(request: urllib.request.Request, *, max_bytes: int, purpose: str) -> bytes:
|
||||||
|
try:
|
||||||
|
with RUNTIME_HTTP_OPENER.open(request, timeout=HTTP_TIMEOUT_SECONDS) as response:
|
||||||
|
if response.getcode() != 200:
|
||||||
|
raise SystemExit(f"{purpose} request failed.")
|
||||||
|
body = response.read(max_bytes + 1)
|
||||||
|
except SystemExit:
|
||||||
|
raise
|
||||||
|
# This is a credential-bearing boundary. Never propagate provider, proxy,
|
||||||
|
# TLS, or test-double exception text because it can contain bearer data.
|
||||||
|
except Exception:
|
||||||
|
raise SystemExit(f"{purpose} request failed.") from None
|
||||||
|
if not isinstance(body, bytes) or len(body) > max_bytes:
|
||||||
|
raise SystemExit(f"{purpose} response was invalid.")
|
||||||
|
return body
|
||||||
|
|
||||||
|
|
||||||
|
def _strict_json_object(body: bytes, *, purpose: str) -> dict[str, Any]:
|
||||||
|
def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||||
|
value: dict[str, Any] = {}
|
||||||
|
for key, item in pairs:
|
||||||
|
if key in value:
|
||||||
|
raise ValueError("duplicate JSON key")
|
||||||
|
value[key] = item
|
||||||
|
return value
|
||||||
|
|
||||||
|
try:
|
||||||
|
decoded = body.decode("utf-8", errors="strict")
|
||||||
|
value = json.loads(
|
||||||
|
decoded,
|
||||||
|
object_pairs_hook=reject_duplicate_keys,
|
||||||
|
parse_constant=lambda _constant: (_ for _ in ()).throw(ValueError("non-finite JSON value")),
|
||||||
|
)
|
||||||
|
except (UnicodeDecodeError, ValueError, TypeError):
|
||||||
|
raise SystemExit(f"{purpose} response was invalid.") from None
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise SystemExit(f"{purpose} response was invalid.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _metadata_request(url: str, *, max_bytes: int, purpose: str) -> bytes:
|
||||||
|
request = urllib.request.Request(url, headers=METADATA_HEADERS, method="GET")
|
||||||
|
return _read_http_response(request, max_bytes=max_bytes, purpose=purpose)
|
||||||
|
|
||||||
|
|
||||||
|
def runtime_password_from_metadata() -> str:
|
||||||
|
email_body = _metadata_request(
|
||||||
|
GCE_METADATA_EMAIL_URL,
|
||||||
|
max_bytes=MAX_METADATA_EMAIL_BYTES,
|
||||||
|
purpose="Runtime metadata identity",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
service_account = email_body.decode("ascii", errors="strict")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
raise SystemExit("Runtime metadata identity response was invalid.") from None
|
||||||
|
if service_account != RUNTIME_SERVICE_ACCOUNT:
|
||||||
|
raise SystemExit("Runtime metadata identity did not match the required service account.")
|
||||||
|
|
||||||
|
token_value = _strict_json_object(
|
||||||
|
_metadata_request(
|
||||||
|
GCE_METADATA_TOKEN_URL,
|
||||||
|
max_bytes=MAX_METADATA_TOKEN_BYTES,
|
||||||
|
purpose="Runtime metadata token",
|
||||||
|
),
|
||||||
|
purpose="Runtime metadata token",
|
||||||
|
)
|
||||||
|
access_token = token_value.get("access_token")
|
||||||
|
token_type = token_value.get("token_type")
|
||||||
|
expires_in = token_value.get("expires_in")
|
||||||
|
if (
|
||||||
|
token_type != "Bearer"
|
||||||
|
or not isinstance(access_token, str)
|
||||||
|
or not re.fullmatch(r"[!-~]{1,4096}", access_token)
|
||||||
|
or not isinstance(expires_in, int)
|
||||||
|
or isinstance(expires_in, bool)
|
||||||
|
or expires_in <= 0
|
||||||
|
):
|
||||||
|
raise SystemExit("Runtime metadata token response was invalid.")
|
||||||
|
|
||||||
|
secret_request = urllib.request.Request(
|
||||||
|
RUNTIME_SECRET_ACCESS_URL,
|
||||||
|
headers={"Accept": "application/json", "Authorization": f"Bearer {access_token}"},
|
||||||
|
method="GET",
|
||||||
|
)
|
||||||
|
secret_value = _strict_json_object(
|
||||||
|
_read_http_response(
|
||||||
|
secret_request,
|
||||||
|
max_bytes=MAX_SECRET_RESPONSE_BYTES,
|
||||||
|
purpose="Scoped runtime secret access",
|
||||||
|
),
|
||||||
|
purpose="Scoped runtime secret access",
|
||||||
|
)
|
||||||
|
payload = secret_value.get("payload")
|
||||||
|
encoded_secret = payload.get("data") if isinstance(payload, dict) else None
|
||||||
|
if not isinstance(encoded_secret, str):
|
||||||
|
raise SystemExit("Scoped runtime secret response was invalid.")
|
||||||
|
try:
|
||||||
|
credential_bytes = base64.b64decode(encoded_secret, validate=True)
|
||||||
|
except (binascii.Error, ValueError):
|
||||||
|
raise SystemExit("Scoped runtime secret response was invalid.") from None
|
||||||
|
if not credential_bytes or len(credential_bytes) > MAX_RUNTIME_SECRET_BYTES:
|
||||||
|
raise SystemExit("Scoped runtime secret response was invalid.")
|
||||||
|
if any(byte in credential_bytes for byte in (0, 10, 13)):
|
||||||
|
raise SystemExit("Scoped runtime secret response was invalid.")
|
||||||
|
try:
|
||||||
|
return credential_bytes.decode("utf-8", errors="strict")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
raise SystemExit("Scoped runtime secret response was invalid.") from None
|
||||||
|
|
||||||
|
|
||||||
def password(args: argparse.Namespace) -> str:
|
def password(args: argparse.Namespace) -> str:
|
||||||
validate_runtime_connection(args)
|
validate_runtime_connection(args)
|
||||||
if args.credential_mode == "clone-readonly" and os.environ.get("PGPASSWORD"):
|
if args.credential_mode == "runtime":
|
||||||
|
return runtime_password_from_metadata()
|
||||||
|
if os.environ.get("PGPASSWORD"):
|
||||||
return os.environ["PGPASSWORD"]
|
return os.environ["PGPASSWORD"]
|
||||||
if not args.password_secret:
|
raise SystemExit("Clone credential mode requires an inherited test credential.")
|
||||||
raise SystemExit("Scoped Cloud SQL password secret is not configured.")
|
|
||||||
result = subprocess.run(
|
|
||||||
[
|
|
||||||
"gcloud",
|
|
||||||
"secrets",
|
|
||||||
"versions",
|
|
||||||
"access",
|
|
||||||
"latest",
|
|
||||||
f"--secret={args.password_secret}",
|
|
||||||
f"--project={args.project}",
|
|
||||||
],
|
|
||||||
text=True,
|
|
||||||
capture_output=True,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
if result.returncode != 0:
|
|
||||||
raise SystemExit(f"Secret access failed: {result.stderr.strip()}")
|
|
||||||
credential = result.stdout.strip()
|
|
||||||
if not credential:
|
|
||||||
raise SystemExit("Scoped Cloud SQL password secret resolved to an empty value.")
|
|
||||||
return credential
|
|
||||||
|
|
||||||
|
|
||||||
def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str:
|
def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str:
|
||||||
env = os.environ.copy()
|
credential = password(args)
|
||||||
for key in tuple(env):
|
if args.credential_mode == "runtime":
|
||||||
if key.startswith("PG"):
|
env = dict(RUNTIME_PSQL_ENV_BASE)
|
||||||
env.pop(key, None)
|
else:
|
||||||
|
env = {key: value for key, value in os.environ.items() if not key.startswith("PG")}
|
||||||
env.update(
|
env.update(
|
||||||
{
|
{
|
||||||
"PGHOST": str(args.host),
|
"PGHOST": str(args.host),
|
||||||
"PGPORT": str(args.port),
|
"PGPORT": str(args.port),
|
||||||
"PGDATABASE": str(db or args.db),
|
"PGDATABASE": str(db or args.db),
|
||||||
"PGUSER": str(args.user),
|
"PGUSER": str(args.user),
|
||||||
"PGSSLMODE": "require",
|
"PGPASSWORD": credential,
|
||||||
"PGCONNECT_TIMEOUT": "8",
|
|
||||||
"PGPASSWORD": password(args),
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
if args.credential_mode == "clone-readonly":
|
if args.credential_mode == "clone-readonly":
|
||||||
|
|
@ -360,9 +537,7 @@ def run_psql(args: argparse.Namespace, sql: str, db: str | None = None) -> str:
|
||||||
check=False,
|
check=False,
|
||||||
)
|
)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
raise SystemExit(
|
raise SystemExit(f"Cloud SQL query failed with exit status {result.returncode}.")
|
||||||
f"Cloud SQL query failed ({result.returncode})\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
|
|
||||||
)
|
|
||||||
return result.stdout
|
return result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -385,6 +560,10 @@ def database_read_marker(args: argparse.Namespace) -> dict[str, Any]:
|
||||||
select jsonb_build_object(
|
select jsonb_build_object(
|
||||||
'database', current_database(),
|
'database', current_database(),
|
||||||
'database_user', current_user,
|
'database_user', current_user,
|
||||||
|
'server_addr', inet_server_addr()::text,
|
||||||
|
'server_port', inet_server_port(),
|
||||||
|
'ssl', coalesce((select ssl from pg_catalog.pg_stat_ssl where pid = pg_backend_pid()), false),
|
||||||
|
'ssl_version', (select version from pg_catalog.pg_stat_ssl where pid = pg_backend_pid()),
|
||||||
'system_identifier', (select system_identifier::text from pg_control_system()),
|
'system_identifier', (select system_identifier::text from pg_control_system()),
|
||||||
'wal_lsn', pg_current_wal_lsn()::text
|
'wal_lsn', pg_current_wal_lsn()::text
|
||||||
)::text;
|
)::text;
|
||||||
|
|
@ -392,12 +571,34 @@ select jsonb_build_object(
|
||||||
rows = psql_json_lines(args, sql, db=args.canonical_db)
|
rows = psql_json_lines(args, sql, db=args.canonical_db)
|
||||||
if len(rows) != 1:
|
if len(rows) != 1:
|
||||||
raise SystemExit("Canonical database read marker returned an invalid row count.")
|
raise SystemExit("Canonical database read marker returned an invalid row count.")
|
||||||
return rows[0]
|
marker = rows[0]
|
||||||
|
expected_database = str(args.canonical_db or "")
|
||||||
|
expected_user = str(args.user or "")
|
||||||
|
if marker.get("database") != expected_database or marker.get("database_user") != expected_user:
|
||||||
|
raise SystemExit("Canonical database read marker returned an unexpected database identity.")
|
||||||
|
if getattr(args, "credential_mode", "runtime") == "runtime":
|
||||||
|
if marker.get("server_addr") != RUNTIME_CLOUDSQL_HOST:
|
||||||
|
raise SystemExit("Canonical database read marker returned an unexpected server address.")
|
||||||
|
if str(marker.get("server_port")) != RUNTIME_CLOUDSQL_PORT:
|
||||||
|
raise SystemExit("Canonical database read marker returned an unexpected server port.")
|
||||||
|
if marker.get("ssl") is not True:
|
||||||
|
raise SystemExit("Canonical database read marker did not prove an encrypted connection.")
|
||||||
|
return marker
|
||||||
|
|
||||||
|
|
||||||
def _read_marker_stable(before: dict[str, Any], after: dict[str, Any]) -> bool:
|
def _read_marker_stable(before: dict[str, Any], after: dict[str, Any]) -> bool:
|
||||||
return all(
|
return all(
|
||||||
before.get(key) == after.get(key) for key in ("database", "database_user", "system_identifier", "wal_lsn")
|
before.get(key) == after.get(key)
|
||||||
|
for key in (
|
||||||
|
"database",
|
||||||
|
"database_user",
|
||||||
|
"server_addr",
|
||||||
|
"server_port",
|
||||||
|
"ssl",
|
||||||
|
"ssl_version",
|
||||||
|
"system_identifier",
|
||||||
|
"wal_lsn",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -470,6 +671,10 @@ def build_retrieval_receipt(
|
||||||
"attempts": attempts,
|
"attempts": attempts,
|
||||||
"database": before.get("database"),
|
"database": before.get("database"),
|
||||||
"database_user": before.get("database_user"),
|
"database_user": before.get("database_user"),
|
||||||
|
"server_addr": before.get("server_addr"),
|
||||||
|
"server_port": before.get("server_port"),
|
||||||
|
"ssl": before.get("ssl"),
|
||||||
|
"ssl_version": before.get("ssl_version"),
|
||||||
"system_identifier": before.get("system_identifier"),
|
"system_identifier": before.get("system_identifier"),
|
||||||
"wal_lsn_before": before.get("wal_lsn"),
|
"wal_lsn_before": before.get("wal_lsn"),
|
||||||
"wal_lsn_after": after.get("wal_lsn"),
|
"wal_lsn_after": after.get("wal_lsn"),
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,15 @@
|
||||||
#!/usr/bin/env bash
|
#!/bin/bash
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
PROFILE_DIR=${HERMES_HOME:-/home/teleo/.hermes/profiles/leoclean}
|
PROFILE_DIR=${HERMES_HOME:-/home/teleo/.hermes/profiles/leoclean}
|
||||||
KB_TOOL="$PROFILE_DIR/bin/kb_tool.py"
|
KB_TOOL="$PROFILE_DIR/bin/kb_tool.py"
|
||||||
CLOUDSQL_TOOL="$PROFILE_DIR/bin/cloudsql_memory_tool.py"
|
SCRIPT_PATH=${BASH_SOURCE[0]}
|
||||||
|
SCRIPT_DIR=${SCRIPT_PATH%/*}
|
||||||
|
if [ "$SCRIPT_DIR" = "$SCRIPT_PATH" ]; then
|
||||||
|
SCRIPT_DIR=.
|
||||||
|
fi
|
||||||
|
SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"
|
||||||
|
CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py"
|
||||||
MODE=${TELEO_KB_MODE:-auto}
|
MODE=${TELEO_KB_MODE:-auto}
|
||||||
COMMAND=${1:-}
|
COMMAND=${1:-}
|
||||||
|
|
||||||
|
|
@ -31,17 +37,22 @@ run_cloudsql() {
|
||||||
echo "teleo-kb: Cloud SQL tool is missing or not executable: $CLOUDSQL_TOOL" >&2
|
echo "teleo-kb: Cloud SQL tool is missing or not executable: $CLOUDSQL_TOOL" >&2
|
||||||
exit 69
|
exit 69
|
||||||
fi
|
fi
|
||||||
for dependency in python3 psql gcloud; do
|
if [ ! -x /usr/bin/python3 ]; then
|
||||||
if ! command -v "$dependency" >/dev/null 2>&1; then
|
echo "teleo-kb: required Cloud SQL dependency is unavailable: /usr/bin/python3" >&2
|
||||||
echo "teleo-kb: required Cloud SQL dependency is unavailable: $dependency" >&2
|
exit 69
|
||||||
exit 69
|
fi
|
||||||
fi
|
if ! command -v psql >/dev/null 2>&1; then
|
||||||
done
|
echo "teleo-kb: required Cloud SQL dependency is unavailable: psql" >&2
|
||||||
|
exit 69
|
||||||
|
fi
|
||||||
extra_args=()
|
extra_args=()
|
||||||
while IFS= read -r arg; do
|
while IFS= read -r arg; do
|
||||||
[ -n "$arg" ] && extra_args+=("$arg")
|
[ -n "$arg" ] && extra_args+=("$arg")
|
||||||
done < <(cloudsql_args)
|
done < <(cloudsql_args)
|
||||||
exec python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}"
|
if [ "${#extra_args[@]}" -gt 0 ]; then
|
||||||
|
exec /usr/bin/python3 "$CLOUDSQL_TOOL" "$@" "${extra_args[@]}"
|
||||||
|
fi
|
||||||
|
exec /usr/bin/python3 "$CLOUDSQL_TOOL" "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
case "$MODE" in
|
case "$MODE" in
|
||||||
|
|
@ -49,10 +60,10 @@ case "$MODE" in
|
||||||
run_cloudsql "$@"
|
run_cloudsql "$@"
|
||||||
;;
|
;;
|
||||||
local)
|
local)
|
||||||
exec python3 "$KB_TOOL" --local "$@"
|
exec /usr/bin/python3 "$KB_TOOL" --local "$@"
|
||||||
;;
|
;;
|
||||||
remote)
|
remote)
|
||||||
exec python3 "$KB_TOOL" "$@"
|
exec /usr/bin/python3 "$KB_TOOL" "$@"
|
||||||
;;
|
;;
|
||||||
auto)
|
auto)
|
||||||
# Canonical commands must fail closed on Cloud SQL errors. Falling through
|
# Canonical commands must fail closed on Cloud SQL errors. Falling through
|
||||||
|
|
@ -62,9 +73,9 @@ case "$MODE" in
|
||||||
run_cloudsql "$@"
|
run_cloudsql "$@"
|
||||||
fi
|
fi
|
||||||
if command -v docker >/dev/null 2>&1; then
|
if command -v docker >/dev/null 2>&1; then
|
||||||
exec python3 "$KB_TOOL" --local "$@"
|
exec /usr/bin/python3 "$KB_TOOL" --local "$@"
|
||||||
fi
|
fi
|
||||||
exec python3 "$KB_TOOL" "$@"
|
exec /usr/bin/python3 "$KB_TOOL" "$@"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Unsupported TELEO_KB_MODE=$MODE; expected auto, cloudsql, local, or remote" >&2
|
echo "Unsupported TELEO_KB_MODE=$MODE; expected auto, cloudsql, local, or remote" >&2
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,4 @@
|
||||||
\set ON_ERROR_STOP on
|
\set ON_ERROR_STOP on
|
||||||
\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD
|
|
||||||
|
|
||||||
-- One-time GCP staging provisioning for Leo's Cloud SQL identity. The password
|
-- One-time GCP staging provisioning for Leo's Cloud SQL identity. The password
|
||||||
-- is supplied only through the process environment and is never stored here.
|
-- is supplied only through the process environment and is never stored here.
|
||||||
|
|
@ -8,7 +7,10 @@
|
||||||
-- through one narrowly-scoped SECURITY DEFINER function. It cannot inherit or
|
-- through one narrowly-scoped SECURITY DEFINER function. It cannot inherit or
|
||||||
-- SET ROLE into another principal, forge the proposer identity, write directly
|
-- SET ROLE into another principal, forge the proposer identity, write directly
|
||||||
-- to canonical/staging tables, or inspect the legacy teleo_restore audit schema.
|
-- to canonical/staging tables, or inspect the legacy teleo_restore audit schema.
|
||||||
select 'create role leoclean_kb_runtime login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8'
|
-- Keep the runtime unable to authenticate until both database phases have
|
||||||
|
-- completed and verified. Any failure before the final transaction therefore
|
||||||
|
-- leaves a disabled role instead of a partially provisioned login.
|
||||||
|
select 'create role leoclean_kb_runtime nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8'
|
||||||
where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_runtime')
|
where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_runtime')
|
||||||
\gexec
|
\gexec
|
||||||
|
|
||||||
|
|
@ -16,15 +18,19 @@ select 'create role leoclean_kb_stage_owner nologin nosuperuser nocreatedb nocre
|
||||||
where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_stage_owner')
|
where not exists (select 1 from pg_catalog.pg_roles where rolname = 'leoclean_kb_stage_owner')
|
||||||
\gexec
|
\gexec
|
||||||
|
|
||||||
select format('alter role leoclean_kb_runtime password %L', :'runtime_password')
|
|
||||||
\gexec
|
|
||||||
|
|
||||||
alter role leoclean_kb_runtime
|
alter role leoclean_kb_runtime
|
||||||
with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8;
|
with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8;
|
||||||
alter role leoclean_kb_stage_owner
|
alter role leoclean_kb_stage_owner
|
||||||
with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1
|
with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1
|
||||||
password null;
|
password null;
|
||||||
|
|
||||||
|
-- Rotate the credential only after an idempotent rerun has disabled any
|
||||||
|
-- pre-existing LOGIN role. This ordering has an intentional autocommit
|
||||||
|
-- boundary: interruption can leave NOLOGIN, never a still-enabled old role.
|
||||||
|
\getenv runtime_password TELEO_LEOCLEAN_DB_PASSWORD
|
||||||
|
select format('alter role leoclean_kb_runtime password %L', :'runtime_password')
|
||||||
|
\gexec
|
||||||
|
|
||||||
alter role leoclean_kb_runtime reset all;
|
alter role leoclean_kb_runtime reset all;
|
||||||
alter role leoclean_kb_stage_owner reset all;
|
alter role leoclean_kb_stage_owner reset all;
|
||||||
|
|
||||||
|
|
@ -261,7 +267,7 @@ begin
|
||||||
select 1
|
select 1
|
||||||
from pg_catalog.pg_roles role_row
|
from pg_catalog.pg_roles role_row
|
||||||
where role_row.rolname = 'leoclean_kb_runtime'
|
where role_row.rolname = 'leoclean_kb_runtime'
|
||||||
and (not role_row.rolcanlogin
|
and (role_row.rolcanlogin
|
||||||
or role_row.rolsuper
|
or role_row.rolsuper
|
||||||
or role_row.rolcreatedb
|
or role_row.rolcreatedb
|
||||||
or role_row.rolcreaterole
|
or role_row.rolcreaterole
|
||||||
|
|
@ -270,7 +276,7 @@ begin
|
||||||
or role_row.rolbypassrls
|
or role_row.rolbypassrls
|
||||||
or role_row.rolconnlimit <> 8)
|
or role_row.rolconnlimit <> 8)
|
||||||
) then
|
) then
|
||||||
raise exception 'leoclean_kb_runtime has unsafe role attributes';
|
raise exception 'leoclean_kb_runtime has unsafe provisioning role attributes';
|
||||||
end if;
|
end if;
|
||||||
|
|
||||||
if exists (
|
if exists (
|
||||||
|
|
@ -818,6 +824,36 @@ commit;
|
||||||
|
|
||||||
\connect teleo_canonical
|
\connect teleo_canonical
|
||||||
|
|
||||||
|
-- LOGIN is the last state change. If this verification fails, the transaction
|
||||||
|
-- rolls back and the role remains NOLOGIN while prior least-privilege grants
|
||||||
|
-- stay inert for direct authentication.
|
||||||
|
begin;
|
||||||
|
|
||||||
|
alter role leoclean_kb_runtime
|
||||||
|
with login nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit 8;
|
||||||
|
|
||||||
|
do $login_verification$
|
||||||
|
begin
|
||||||
|
if exists (
|
||||||
|
select 1
|
||||||
|
from pg_catalog.pg_roles role_row
|
||||||
|
where role_row.rolname = 'leoclean_kb_runtime'
|
||||||
|
and (not role_row.rolcanlogin
|
||||||
|
or role_row.rolsuper
|
||||||
|
or role_row.rolcreatedb
|
||||||
|
or role_row.rolcreaterole
|
||||||
|
or role_row.rolinherit
|
||||||
|
or role_row.rolreplication
|
||||||
|
or role_row.rolbypassrls
|
||||||
|
or role_row.rolconnlimit <> 8)
|
||||||
|
) then
|
||||||
|
raise exception 'leoclean_kb_runtime final login attributes are unsafe';
|
||||||
|
end if;
|
||||||
|
end
|
||||||
|
$login_verification$;
|
||||||
|
|
||||||
|
commit;
|
||||||
|
|
||||||
select jsonb_build_object(
|
select jsonb_build_object(
|
||||||
'role', rolname,
|
'role', rolname,
|
||||||
'superuser', rolsuper,
|
'superuser', rolsuper,
|
||||||
|
|
@ -835,6 +871,7 @@ select jsonb_build_object(
|
||||||
'can_select_claims', has_table_privilege(rolname, 'public.claims', 'SELECT'),
|
'can_select_claims', has_table_privilege(rolname, 'public.claims', 'SELECT'),
|
||||||
'can_select_proposals', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'SELECT'),
|
'can_select_proposals', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'SELECT'),
|
||||||
'can_insert_proposals_directly', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'INSERT'),
|
'can_insert_proposals_directly', has_table_privilege(rolname, 'kb_stage.kb_proposals', 'INSERT'),
|
||||||
|
'can_connect_legacy_database', has_database_privilege(rolname, 'teleo_kb', 'CONNECT'),
|
||||||
'can_create_database_objects', has_database_privilege(rolname, 'teleo_canonical', 'CREATE'),
|
'can_create_database_objects', has_database_privilege(rolname, 'teleo_canonical', 'CREATE'),
|
||||||
'can_create_temp_objects', has_database_privilege(rolname, 'teleo_canonical', 'TEMP'),
|
'can_create_temp_objects', has_database_privilege(rolname, 'teleo_canonical', 'TEMP'),
|
||||||
'can_stage_proposal', has_function_privilege(
|
'can_stage_proposal', has_function_privilege(
|
||||||
|
|
|
||||||
1838
ops/verify_gcp_leoclean_runtime_permissions.py
Normal file
1838
ops/verify_gcp_leoclean_runtime_permissions.py
Normal file
File diff suppressed because it is too large
Load diff
219
ops/verify_gcp_leoclean_service_environment.py
Executable file
219
ops/verify_gcp_leoclean_service_environment.py
Executable file
|
|
@ -0,0 +1,219 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Verify the effective Leo service environment without exposing its values."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
PROJECT_ID = "teleo-501523"
|
||||||
|
PRIVATE_CLOUDSQL_HOST = "10.61.0.3"
|
||||||
|
PRIVATE_CLOUDSQL_PORT = "5432"
|
||||||
|
CANONICAL_DATABASE = "teleo_canonical"
|
||||||
|
RUNTIME_DATABASE_ROLE = "leoclean_kb_runtime"
|
||||||
|
SCOPED_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password"
|
||||||
|
ADMINISTRATOR_PASSWORD_SECRET = "gcp-teleo-pgvector-standby-postgres-password"
|
||||||
|
RUNTIME_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||||
|
|
||||||
|
FORBIDDEN_EXACT_FIELDS = frozenset(
|
||||||
|
{
|
||||||
|
"ALL_PROXY",
|
||||||
|
"BASH_ENV",
|
||||||
|
"BASHOPTS",
|
||||||
|
"CDPATH",
|
||||||
|
"CURL_CA_BUNDLE",
|
||||||
|
"ENV",
|
||||||
|
"GLOBIGNORE",
|
||||||
|
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||||
|
"HTTPS_PROXY",
|
||||||
|
"HTTP_PROXY",
|
||||||
|
"IFS",
|
||||||
|
"NO_PROXY",
|
||||||
|
"PS4",
|
||||||
|
"REQUESTS_CA_BUNDLE",
|
||||||
|
"SHELLOPTS",
|
||||||
|
"SSL_CERT_DIR",
|
||||||
|
"SSL_CERT_FILE",
|
||||||
|
"SSLKEYLOGFILE",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class VerificationError(RuntimeError):
|
||||||
|
"""A sanitized failure safe to serialize in a public receipt."""
|
||||||
|
|
||||||
|
def __init__(self, code: str, fields: tuple[str, ...] = ()) -> None:
|
||||||
|
self.code = code
|
||||||
|
self.fields = tuple(sorted(set(fields)))
|
||||||
|
super().__init__(code)
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
payload: dict[str, object] = {"code": self.code}
|
||||||
|
if self.fields:
|
||||||
|
payload["fields"] = list(self.fields)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
class _SanitizedArgumentParser(argparse.ArgumentParser):
|
||||||
|
def error(self, _message: str) -> None:
|
||||||
|
raise VerificationError("invalid_arguments")
|
||||||
|
|
||||||
|
|
||||||
|
def validate_pid(value: str | int) -> int:
|
||||||
|
"""Return a positive process id without retaining the input on failure."""
|
||||||
|
|
||||||
|
if isinstance(value, bool) or not isinstance(value, (str, int)):
|
||||||
|
raise VerificationError("invalid_pid")
|
||||||
|
if isinstance(value, str) and (not value.isascii() or not value.isdecimal() or value.startswith("0")):
|
||||||
|
raise VerificationError("invalid_pid")
|
||||||
|
try:
|
||||||
|
pid = int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
raise VerificationError("invalid_pid") from None
|
||||||
|
if pid <= 0:
|
||||||
|
raise VerificationError("invalid_pid")
|
||||||
|
return pid
|
||||||
|
|
||||||
|
|
||||||
|
def expected_environment() -> dict[str, str]:
|
||||||
|
"""Build the exact reviewed environment contract."""
|
||||||
|
|
||||||
|
return {
|
||||||
|
"PATH": RUNTIME_PATH,
|
||||||
|
"TELEO_CANONICAL_CLOUDSQL_DB": CANONICAL_DATABASE,
|
||||||
|
"TELEO_CLOUDSQL_DB": CANONICAL_DATABASE,
|
||||||
|
"TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK": "0",
|
||||||
|
"TELEO_CLOUDSQL_HOST": PRIVATE_CLOUDSQL_HOST,
|
||||||
|
"TELEO_CLOUDSQL_PASSWORD_SECRET": SCOPED_PASSWORD_SECRET,
|
||||||
|
"TELEO_CLOUDSQL_PORT": PRIVATE_CLOUDSQL_PORT,
|
||||||
|
"TELEO_CLOUDSQL_USER": RUNTIME_DATABASE_ROLE,
|
||||||
|
"TELEO_GCP_METADATA_ONLY": "1",
|
||||||
|
"TELEO_GCP_PROJECT": PROJECT_ID,
|
||||||
|
"TELEO_KB_MODE": "cloudsql",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_environment(raw: bytes) -> dict[str, str]:
|
||||||
|
"""Parse a Linux ``/proc/PID/environ`` block and reject ambiguity."""
|
||||||
|
|
||||||
|
if not isinstance(raw, bytes) or not raw or not raw.endswith(b"\0"):
|
||||||
|
raise VerificationError("environment_malformed")
|
||||||
|
|
||||||
|
records = raw[:-1].split(b"\0")
|
||||||
|
if any(not record for record in records):
|
||||||
|
raise VerificationError("environment_malformed")
|
||||||
|
|
||||||
|
environment: dict[str, str] = {}
|
||||||
|
try:
|
||||||
|
for record in records:
|
||||||
|
encoded_key, separator, encoded_value = record.partition(b"=")
|
||||||
|
if not separator or not encoded_key:
|
||||||
|
raise VerificationError("environment_malformed")
|
||||||
|
key = encoded_key.decode("utf-8", "strict")
|
||||||
|
value = encoded_value.decode("utf-8", "strict")
|
||||||
|
if key in environment:
|
||||||
|
raise VerificationError("environment_malformed")
|
||||||
|
environment[key] = value
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
raise VerificationError("environment_malformed") from None
|
||||||
|
return environment
|
||||||
|
|
||||||
|
|
||||||
|
def _forbidden_field_labels(environment: Mapping[str, str]) -> tuple[str, ...]:
|
||||||
|
labels: set[str] = set()
|
||||||
|
for key in environment:
|
||||||
|
normalized = key.upper()
|
||||||
|
if normalized.startswith("PG"):
|
||||||
|
labels.add("PG*")
|
||||||
|
if normalized.startswith("CLOUDSDK_"):
|
||||||
|
labels.add("CLOUDSDK_*")
|
||||||
|
if normalized.startswith("LD_"):
|
||||||
|
labels.add("LD_*")
|
||||||
|
if normalized.startswith("BASH_FUNC_"):
|
||||||
|
labels.add("BASH_FUNC_*")
|
||||||
|
if normalized.startswith("PYTHON") and normalized != "PYTHONUNBUFFERED":
|
||||||
|
labels.add("PYTHON*")
|
||||||
|
if normalized in FORBIDDEN_EXACT_FIELDS:
|
||||||
|
labels.add(normalized)
|
||||||
|
return tuple(sorted(labels))
|
||||||
|
|
||||||
|
|
||||||
|
def validate_environment(environment: Mapping[str, str]) -> tuple[str, ...]:
|
||||||
|
"""Validate the effective environment and return only reviewed field names."""
|
||||||
|
|
||||||
|
if not isinstance(environment, Mapping) or any(
|
||||||
|
not isinstance(key, str) or not isinstance(value, str) for key, value in environment.items()
|
||||||
|
):
|
||||||
|
raise VerificationError("environment_malformed")
|
||||||
|
|
||||||
|
expected = expected_environment()
|
||||||
|
administrator_secret = ADMINISTRATOR_PASSWORD_SECRET.casefold()
|
||||||
|
if any(administrator_secret in value.casefold() for value in environment.values()):
|
||||||
|
raise VerificationError("administrator_secret_reference")
|
||||||
|
|
||||||
|
forbidden_fields = _forbidden_field_labels(environment)
|
||||||
|
if forbidden_fields:
|
||||||
|
raise VerificationError("forbidden_environment_fields", forbidden_fields)
|
||||||
|
|
||||||
|
mismatched = tuple(
|
||||||
|
sorted(key for key, expected_value in expected.items() if environment.get(key) != expected_value)
|
||||||
|
)
|
||||||
|
if mismatched:
|
||||||
|
raise VerificationError("environment_mismatch", mismatched)
|
||||||
|
|
||||||
|
return tuple(sorted(expected))
|
||||||
|
|
||||||
|
|
||||||
|
def verify_process_environment(
|
||||||
|
pid: int,
|
||||||
|
*,
|
||||||
|
proc_root: Path = Path("/proc"),
|
||||||
|
) -> dict[str, object]:
|
||||||
|
"""Read and verify one process environment, returning a sanitized receipt."""
|
||||||
|
|
||||||
|
validated_pid = validate_pid(pid)
|
||||||
|
raw = (proc_root / str(validated_pid) / "environ").read_bytes()
|
||||||
|
validated_fields = validate_environment(parse_environment(raw))
|
||||||
|
return {
|
||||||
|
"runtime_environment": "scoped",
|
||||||
|
"status": "pass",
|
||||||
|
"validated_fields": list(validated_fields),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def failure_receipt(error: VerificationError) -> dict[str, object]:
|
||||||
|
return {"error": error.as_dict(), "status": "fail"}
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_json(payload: Mapping[str, Any]) -> str:
|
||||||
|
return json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||||
|
parser = _SanitizedArgumentParser(description=__doc__, add_help=False)
|
||||||
|
parser.add_argument("--pid", required=True)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
try:
|
||||||
|
args = parse_args(argv)
|
||||||
|
receipt = verify_process_environment(validate_pid(args.pid))
|
||||||
|
returncode = 0
|
||||||
|
except VerificationError as exc:
|
||||||
|
receipt = failure_receipt(exc)
|
||||||
|
returncode = 1
|
||||||
|
except Exception:
|
||||||
|
receipt = failure_receipt(VerificationError("internal_error"))
|
||||||
|
returncode = 1
|
||||||
|
|
||||||
|
sys.stdout.write(canonical_json(receipt))
|
||||||
|
return returncode
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -47,7 +47,7 @@ DEFAULT_HOST = "10.61.0.3"
|
||||||
DEFAULT_PROJECT = "teleo-501523"
|
DEFAULT_PROJECT = "teleo-501523"
|
||||||
DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password"
|
DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password"
|
||||||
DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql"
|
DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql"
|
||||||
REVIEWED_CLOUDSQL_TOOL_SHA256 = "2617bb501456758131ff41a6c77b3c462e21c28c4348da319b60defb051c7c87"
|
REVIEWED_CLOUDSQL_TOOL_SHA256 = "66fa84b3bd6fe67e102fbce4509ea28c3537ba5bb1091e1a1a457b7425366064"
|
||||||
REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a"
|
REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a"
|
||||||
COUNT_READBACK_RE = re.compile(
|
COUNT_READBACK_RE = re.compile(
|
||||||
r"DB readback:\s*claims:\s*`?(?P<claims>\d+)`?;\s*"
|
r"DB readback:\s*claims:\s*`?(?P<claims>\d+)`?;\s*"
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,19 @@
|
||||||
[Service]
|
[Service]
|
||||||
UnsetEnvironment=PGPASSWORD
|
UnsetEnvironment=PGPASSWORD
|
||||||
RuntimeDirectory=leoclean-gcp-prod-parallel
|
UnsetEnvironment=PGPASSFILE PGSERVICE PGHOST PGPORT PGDATABASE PGUSER PGOPTIONS
|
||||||
RuntimeDirectoryMode=0700
|
UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN
|
||||||
Environment=CLOUDSDK_CONFIG=/run/leoclean-gcp-prod-parallel
|
UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS CDPATH GLOBIGNORE IFS PS4
|
||||||
|
UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT LD_DEBUG LD_DEBUG_OUTPUT LD_DYNAMIC_WEAK LD_HWCAP_MASK LD_ORIGIN_PATH LD_PROFILE LD_SHOW_AUXV LD_TRACE_LOADED_OBJECTS LD_USE_LOAD_BIAS LD_VERBOSE LD_WARN
|
||||||
|
UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT PYTHONUSERBASE PYTHONHTTPSVERIFY
|
||||||
|
UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy
|
||||||
|
UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE
|
||||||
|
ReadOnlyPaths=/home/teleo/.hermes
|
||||||
|
ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state /home/teleo/.hermes/profiles/leoclean/workspace
|
||||||
|
BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin
|
||||||
|
CapabilityBoundingSet=~CAP_SYS_ADMIN
|
||||||
|
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||||
Environment=TELEO_KB_MODE=cloudsql
|
Environment=TELEO_KB_MODE=cloudsql
|
||||||
|
Environment=TELEO_GCP_METADATA_ONLY=1
|
||||||
Environment=TELEO_GCP_PROJECT=teleo-501523
|
Environment=TELEO_GCP_PROJECT=teleo-501523
|
||||||
Environment=TELEO_CLOUDSQL_HOST=10.61.0.3
|
Environment=TELEO_CLOUDSQL_HOST=10.61.0.3
|
||||||
Environment=TELEO_CLOUDSQL_PORT=5432
|
Environment=TELEO_CLOUDSQL_PORT=5432
|
||||||
|
|
|
||||||
797
tests/test_gcp_leoclean_runtime_permissions.py
Normal file
797
tests/test_gcp_leoclean_runtime_permissions.py
Normal file
|
|
@ -0,0 +1,797 @@
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from ops import verify_gcp_leoclean_runtime_permissions as verifier
|
||||||
|
|
||||||
|
RUN_ID = "20260715-leo-security"
|
||||||
|
SOURCE_REF = f"leo-runtime-permission:{RUN_ID}"
|
||||||
|
SECRET_VALUE = "unit-only-secret-value%42"
|
||||||
|
AGENT_ID = "11111111-1111-4111-8111-111111111111"
|
||||||
|
|
||||||
|
|
||||||
|
class FakeRunner:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
sqlstate_override: tuple[str, str] | None = None,
|
||||||
|
sqlstate_missing_for: str | None = None,
|
||||||
|
unexpected_success_for: str | None = None,
|
||||||
|
scoped_failure: bool = False,
|
||||||
|
administrator_mode: str = "denied",
|
||||||
|
identity_override: Mapping[str, object] | None = None,
|
||||||
|
role_posture_override: Mapping[str, object] | None = None,
|
||||||
|
stage_owner_role_posture_override: Mapping[str, object] | None = None,
|
||||||
|
function_posture_override: Mapping[str, Mapping[str, object]] | None = None,
|
||||||
|
stage_definition_override: Mapping[str, object] | None = None,
|
||||||
|
catalog_posture_override: Mapping[str, object] | None = None,
|
||||||
|
legacy_catalog_posture_override: Mapping[str, object] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.sqlstate_override = sqlstate_override
|
||||||
|
self.sqlstate_missing_for = sqlstate_missing_for
|
||||||
|
self.unexpected_success_for = unexpected_success_for
|
||||||
|
self.scoped_failure = scoped_failure
|
||||||
|
self.administrator_mode = administrator_mode
|
||||||
|
self.identity_override = dict(identity_override or {})
|
||||||
|
self.role_posture_override = dict(role_posture_override or {})
|
||||||
|
self.stage_owner_role_posture_override = dict(stage_owner_role_posture_override or {})
|
||||||
|
self.function_posture_override = {
|
||||||
|
name: dict(values) for name, values in (function_posture_override or {}).items()
|
||||||
|
}
|
||||||
|
self.stage_definition_override = dict(stage_definition_override or {})
|
||||||
|
self.catalog_posture_override = dict(catalog_posture_override or {})
|
||||||
|
self.legacy_catalog_posture_override = dict(legacy_catalog_posture_override or {})
|
||||||
|
self.calls: list[tuple[list[str], dict[str, str], int, bool]] = []
|
||||||
|
self.negative = {check.sql: check for check in verifier._negative_checks(RUN_ID, SOURCE_REF)}
|
||||||
|
|
||||||
|
def __call__(
|
||||||
|
self,
|
||||||
|
command: list[str],
|
||||||
|
env: Mapping[str, str],
|
||||||
|
timeout: int,
|
||||||
|
discard_stdout: bool,
|
||||||
|
) -> verifier.CommandResult:
|
||||||
|
self.calls.append((list(command), dict(env), timeout, discard_stdout))
|
||||||
|
|
||||||
|
if command[0] == verifier.GCLOUD_BIN:
|
||||||
|
scoped = f"--secret={verifier.SCOPED_PASSWORD_SECRET}" in command
|
||||||
|
if scoped:
|
||||||
|
if self.scoped_failure:
|
||||||
|
return verifier.CommandResult(1, SECRET_VALUE, f"scoped failure {SECRET_VALUE}")
|
||||||
|
return verifier.CommandResult(0, f"{SECRET_VALUE}\n", f"ignored warning {SECRET_VALUE}")
|
||||||
|
assert f"--secret={verifier.ADMINISTRATOR_PASSWORD_SECRET}" in command
|
||||||
|
assert discard_stdout is True
|
||||||
|
if self.administrator_mode == "success":
|
||||||
|
return verifier.CommandResult(0, SECRET_VALUE, f"ignored {SECRET_VALUE}")
|
||||||
|
if self.administrator_mode == "wrong-error":
|
||||||
|
return verifier.CommandResult(1, SECRET_VALUE, f"network unavailable {SECRET_VALUE}")
|
||||||
|
return verifier.CommandResult(
|
||||||
|
1,
|
||||||
|
SECRET_VALUE,
|
||||||
|
(
|
||||||
|
"ERROR: PERMISSION_DENIED: Permission "
|
||||||
|
f"'{verifier.IAM_PERMISSION}' denied for the administrator secret; {SECRET_VALUE}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert command[0] == verifier.PSQL_BIN
|
||||||
|
assert discard_stdout is False
|
||||||
|
sql = next(part.removeprefix("--command=") for part in command if part.startswith("--command="))
|
||||||
|
|
||||||
|
if sql == verifier._identity_sql():
|
||||||
|
identity: dict[str, object] = {
|
||||||
|
"current_user": verifier.RUNTIME_DATABASE_ROLE,
|
||||||
|
"database": verifier.CANONICAL_DATABASE,
|
||||||
|
"leak": SECRET_VALUE,
|
||||||
|
"server_addr": verifier.PRIVATE_CLOUDSQL_HOST,
|
||||||
|
"server_port": verifier.PRIVATE_CLOUDSQL_PORT,
|
||||||
|
"session_user": verifier.RUNTIME_DATABASE_ROLE,
|
||||||
|
"ssl": True,
|
||||||
|
"ssl_version": "TLSv1.3",
|
||||||
|
}
|
||||||
|
identity.update(self.identity_override)
|
||||||
|
return verifier.CommandResult(0, json.dumps(identity), f"ignored {SECRET_VALUE}")
|
||||||
|
if sql == verifier._allowed_read_sql():
|
||||||
|
return verifier.CommandResult(
|
||||||
|
0,
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"claims_rows_sampled": 1,
|
||||||
|
"leak": SECRET_VALUE,
|
||||||
|
"proposal_rows_sampled": 0,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
f"ignored {SECRET_VALUE}",
|
||||||
|
)
|
||||||
|
if sql == verifier._role_posture_sql():
|
||||||
|
role_posture: dict[str, object] = {
|
||||||
|
"bypasses_rls": False,
|
||||||
|
"can_create_db": False,
|
||||||
|
"can_create_role": False,
|
||||||
|
"can_login": True,
|
||||||
|
"can_replicate": False,
|
||||||
|
"connection_limit": 8,
|
||||||
|
"direct_membership_edges": 0,
|
||||||
|
"inherits_privileges": False,
|
||||||
|
"is_superuser": False,
|
||||||
|
"leak": SECRET_VALUE,
|
||||||
|
"role": verifier.RUNTIME_DATABASE_ROLE,
|
||||||
|
}
|
||||||
|
role_posture.update(self.role_posture_override)
|
||||||
|
return verifier.CommandResult(0, json.dumps(role_posture), f"ignored {SECRET_VALUE}")
|
||||||
|
if sql == verifier._stage_owner_role_posture_sql():
|
||||||
|
owner_posture: dict[str, object] = {
|
||||||
|
"bypasses_rls": False,
|
||||||
|
"can_create_db": False,
|
||||||
|
"can_create_role": False,
|
||||||
|
"can_login": False,
|
||||||
|
"can_replicate": False,
|
||||||
|
"connection_limit": -1,
|
||||||
|
"direct_membership_edges": 0,
|
||||||
|
"inherits_privileges": False,
|
||||||
|
"is_superuser": False,
|
||||||
|
"leak": SECRET_VALUE,
|
||||||
|
"role": verifier.STAGE_OWNER_DATABASE_ROLE,
|
||||||
|
}
|
||||||
|
owner_posture.update(self.stage_owner_role_posture_override)
|
||||||
|
return verifier.CommandResult(0, json.dumps(owner_posture), f"ignored {SECRET_VALUE}")
|
||||||
|
if sql == verifier._function_privilege_posture_sql():
|
||||||
|
function_posture: dict[str, dict[str, object]] = {
|
||||||
|
name: {
|
||||||
|
"execute": expected_execute,
|
||||||
|
"exists": expected_exists,
|
||||||
|
"leak": SECRET_VALUE,
|
||||||
|
}
|
||||||
|
for name, _signature, expected_exists, expected_execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS
|
||||||
|
}
|
||||||
|
for name, override in self.function_posture_override.items():
|
||||||
|
function_posture.setdefault(name, {}).update(override)
|
||||||
|
function_posture["unexpected_leak"] = {"value": SECRET_VALUE}
|
||||||
|
return verifier.CommandResult(0, json.dumps(function_posture), f"ignored {SECRET_VALUE}")
|
||||||
|
if sql == verifier._stage_function_definition_sql():
|
||||||
|
stage_definition: dict[str, object] = {
|
||||||
|
"acl_exact": True,
|
||||||
|
"argument_modes": None,
|
||||||
|
"configuration": ["search_path=pg_catalog, pg_temp"],
|
||||||
|
"exists": True,
|
||||||
|
"leak": SECRET_VALUE,
|
||||||
|
"runtime_execute": True,
|
||||||
|
"source": verifier.EXPECTED_STAGE_FUNCTION_SOURCE,
|
||||||
|
**dict(verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS),
|
||||||
|
}
|
||||||
|
stage_definition.update(self.stage_definition_override)
|
||||||
|
return verifier.CommandResult(0, json.dumps(stage_definition), f"ignored {SECRET_VALUE}")
|
||||||
|
if sql == verifier._catalog_privilege_posture_sql():
|
||||||
|
catalog_posture: dict[str, object] = {
|
||||||
|
field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS
|
||||||
|
}
|
||||||
|
catalog_posture.update(
|
||||||
|
{
|
||||||
|
"leak": SECRET_VALUE,
|
||||||
|
**{field: True for field in verifier.CATALOG_TRUE_FIELDS},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
catalog_posture.update(self.catalog_posture_override)
|
||||||
|
return verifier.CommandResult(0, json.dumps(catalog_posture), f"ignored {SECRET_VALUE}")
|
||||||
|
if sql == verifier._legacy_catalog_privilege_posture_sql():
|
||||||
|
legacy_posture: dict[str, object] = {
|
||||||
|
field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS
|
||||||
|
}
|
||||||
|
legacy_posture["leak"] = SECRET_VALUE
|
||||||
|
legacy_posture.update(self.legacy_catalog_posture_override)
|
||||||
|
return verifier.CommandResult(0, json.dumps(legacy_posture), f"ignored {SECRET_VALUE}")
|
||||||
|
if sql == verifier._canary_count_sql(SOURCE_REF):
|
||||||
|
return verifier.CommandResult(0, "0\n", f"ignored {SECRET_VALUE}")
|
||||||
|
if sql == verifier._stage_sql(RUN_ID, SOURCE_REF):
|
||||||
|
return verifier.CommandResult(
|
||||||
|
0,
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"agent_id_matches": True,
|
||||||
|
"proposal": {
|
||||||
|
"leak": SECRET_VALUE,
|
||||||
|
"proposed_by_agent_id": AGENT_ID,
|
||||||
|
"proposed_by_handle": "leo",
|
||||||
|
"source_ref": SOURCE_REF,
|
||||||
|
"status": "pending_review",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
),
|
||||||
|
f"ignored {SECRET_VALUE}",
|
||||||
|
)
|
||||||
|
|
||||||
|
check = self.negative[sql]
|
||||||
|
if check.name == self.unexpected_success_for:
|
||||||
|
return verifier.CommandResult(0, SECRET_VALUE, f"unexpected success {SECRET_VALUE}")
|
||||||
|
if check.name == self.sqlstate_missing_for:
|
||||||
|
return verifier.CommandResult(1, SECRET_VALUE, f"denied without state {SECRET_VALUE}")
|
||||||
|
observed = check.expected_sqlstate
|
||||||
|
if self.sqlstate_override is not None and check.name == self.sqlstate_override[0]:
|
||||||
|
observed = self.sqlstate_override[1]
|
||||||
|
return verifier.CommandResult(
|
||||||
|
1,
|
||||||
|
SECRET_VALUE,
|
||||||
|
f"ERROR: {observed}: expected test denial; {SECRET_VALUE}\nLOCATION: test",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def poisoned_environment() -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
"CLOUDSDK_AUTH_ACCESS_TOKEN": "must-not-reach-gcloud",
|
||||||
|
"GOOGLE_APPLICATION_CREDENTIALS": "/tmp/must-not-reach-gcloud.json",
|
||||||
|
"HOME": "/home/teleo",
|
||||||
|
"LANG": "C.UTF-8",
|
||||||
|
"LD_PRELOAD": "/tmp/must-not-load.so",
|
||||||
|
"OTHER": "preserved",
|
||||||
|
"PGDATABASE": "wrong-db",
|
||||||
|
"PGHOST": "public.example.invalid",
|
||||||
|
"PGPASSWORD": "administrator-password",
|
||||||
|
"PGSERVICE": "broad-service",
|
||||||
|
"pgsslmode": "disable",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run_success(runner: FakeRunner) -> dict[str, object]:
|
||||||
|
return verifier.verify_runtime_permissions(
|
||||||
|
RUN_ID,
|
||||||
|
runner=runner,
|
||||||
|
base_env=poisoned_environment(),
|
||||||
|
effective_user="teleo",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def assert_child_boundaries(runner: FakeRunner) -> None:
|
||||||
|
assert runner.calls
|
||||||
|
for command, env, timeout, discard_stdout in runner.calls:
|
||||||
|
assert SECRET_VALUE not in "\n".join(command)
|
||||||
|
assert timeout == verifier.COMMAND_TIMEOUT_SECONDS
|
||||||
|
pg_keys = sorted(key for key in env if key.upper().startswith("PG"))
|
||||||
|
if command[0] == verifier.PSQL_BIN:
|
||||||
|
assert pg_keys == ["PGPASSWORD"]
|
||||||
|
assert env["PGPASSWORD"] == SECRET_VALUE
|
||||||
|
assert discard_stdout is False
|
||||||
|
else:
|
||||||
|
assert pg_keys == []
|
||||||
|
assert "CLOUDSDK_AUTH_ACCESS_TOKEN" not in env
|
||||||
|
assert "GOOGLE_APPLICATION_CREDENTIALS" not in env
|
||||||
|
assert env["HOME"] == "/home/teleo"
|
||||||
|
assert env["LANG"] == "C"
|
||||||
|
assert env["LC_ALL"] == "C"
|
||||||
|
assert env["PATH"] == verifier.CHILD_PATH
|
||||||
|
assert "LD_PRELOAD" not in env
|
||||||
|
assert "OTHER" not in env
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_receipt_contract_is_sanitized_and_uses_exact_child_environments() -> None:
|
||||||
|
runner = FakeRunner()
|
||||||
|
|
||||||
|
receipt = run_success(runner)
|
||||||
|
serialized = verifier.canonical_json(receipt)
|
||||||
|
|
||||||
|
assert receipt["status"] == "pass"
|
||||||
|
assert receipt["current_tier"] == verifier.REQUIRED_TIER
|
||||||
|
assert receipt["database_identity"] == {
|
||||||
|
"current_user": verifier.RUNTIME_DATABASE_ROLE,
|
||||||
|
"database": verifier.CANONICAL_DATABASE,
|
||||||
|
"server_addr": verifier.PRIVATE_CLOUDSQL_HOST,
|
||||||
|
"server_port": verifier.PRIVATE_CLOUDSQL_PORT,
|
||||||
|
"session_user": verifier.RUNTIME_DATABASE_ROLE,
|
||||||
|
"ssl": True,
|
||||||
|
"ssl_version": "TLSv1.3",
|
||||||
|
}
|
||||||
|
assert receipt["checks"]["rolled_back_proposal"] == {
|
||||||
|
"agent_id_matches": True,
|
||||||
|
"proposed_by_agent_id": AGENT_ID,
|
||||||
|
"proposed_by_handle": "leo",
|
||||||
|
"source_ref": SOURCE_REF,
|
||||||
|
"status": "pending_review",
|
||||||
|
"transaction": "rolled_back",
|
||||||
|
}
|
||||||
|
assert receipt["checks"]["canary_rows_before"] == 0
|
||||||
|
assert receipt["checks"]["canary_rows_after"] == 0
|
||||||
|
assert receipt["checks"]["role_posture"] == {
|
||||||
|
"bypasses_rls": False,
|
||||||
|
"can_create_db": False,
|
||||||
|
"can_create_role": False,
|
||||||
|
"can_login": True,
|
||||||
|
"can_replicate": False,
|
||||||
|
"connection_limit": 8,
|
||||||
|
"direct_membership_edges": 0,
|
||||||
|
"inherits_privileges": False,
|
||||||
|
"is_superuser": False,
|
||||||
|
"role": verifier.RUNTIME_DATABASE_ROLE,
|
||||||
|
}
|
||||||
|
assert receipt["checks"]["stage_owner_role_posture"] == {
|
||||||
|
"bypasses_rls": False,
|
||||||
|
"can_create_db": False,
|
||||||
|
"can_create_role": False,
|
||||||
|
"can_login": False,
|
||||||
|
"can_replicate": False,
|
||||||
|
"connection_limit": -1,
|
||||||
|
"direct_membership_edges": 0,
|
||||||
|
"inherits_privileges": False,
|
||||||
|
"is_superuser": False,
|
||||||
|
"role": verifier.STAGE_OWNER_DATABASE_ROLE,
|
||||||
|
}
|
||||||
|
assert receipt["checks"]["function_privileges"] == {
|
||||||
|
name: {
|
||||||
|
"execute": expected_execute,
|
||||||
|
"exists": expected_exists,
|
||||||
|
"signature": signature,
|
||||||
|
}
|
||||||
|
for name, signature, expected_exists, expected_execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS
|
||||||
|
}
|
||||||
|
assert receipt["checks"]["catalog_privileges"] == {
|
||||||
|
**{field: 0 for field in verifier.CATALOG_ZERO_COUNT_FIELDS},
|
||||||
|
**{field: True for field in verifier.CATALOG_TRUE_FIELDS},
|
||||||
|
}
|
||||||
|
assert receipt["checks"]["legacy_catalog_privileges"] == {
|
||||||
|
field: 0 for field in verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS
|
||||||
|
}
|
||||||
|
assert receipt["checks"]["stage_function_definition"] == {
|
||||||
|
"acl_exact": True,
|
||||||
|
"argument_modes": None,
|
||||||
|
"configuration": ["search_path=pg_catalog, pg_temp"],
|
||||||
|
"exists": True,
|
||||||
|
**dict(verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS),
|
||||||
|
"runtime_execute": True,
|
||||||
|
"signature": verifier.STAGE_FUNCTION_SIGNATURE,
|
||||||
|
"source_sha256": verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256,
|
||||||
|
}
|
||||||
|
assert len(receipt["checks"]["negative_permissions"]) == len(verifier._negative_checks(RUN_ID, SOURCE_REF))
|
||||||
|
assert receipt["secret_access"]["administrator"]["classification"] == "iam_permission_denied"
|
||||||
|
assert receipt["secret_access"]["administrator"]["stdout_discarded"] is True
|
||||||
|
assert SECRET_VALUE not in serialized
|
||||||
|
assert "administrator-password" not in serialized
|
||||||
|
assert "stage_leoclean_proposal is restricted" not in serialized
|
||||||
|
assert_child_boundaries(runner)
|
||||||
|
|
||||||
|
administrator_call = runner.calls[-1]
|
||||||
|
assert administrator_call[0][0] == verifier.GCLOUD_BIN
|
||||||
|
assert administrator_call[3] is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("attribute", "unsafe_value"),
|
||||||
|
[
|
||||||
|
("can_login", False),
|
||||||
|
("is_superuser", True),
|
||||||
|
("can_create_db", True),
|
||||||
|
("can_create_role", True),
|
||||||
|
("inherits_privileges", True),
|
||||||
|
("can_replicate", True),
|
||||||
|
("bypasses_rls", True),
|
||||||
|
("connection_limit", 7),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_any_unsafe_runtime_role_attribute_fails_closed(attribute: str, unsafe_value: object) -> None:
|
||||||
|
runner = FakeRunner(role_posture_override={attribute: unsafe_value})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "runtime_role_posture_mismatch"
|
||||||
|
assert caught.value.check == "runtime_role_posture"
|
||||||
|
assert SECRET_VALUE not in str(caught.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_any_direct_runtime_role_membership_edge_fails_closed() -> None:
|
||||||
|
runner = FakeRunner(role_posture_override={"direct_membership_edges": 1})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "runtime_role_posture_mismatch"
|
||||||
|
assert caught.value.check == "runtime_role_posture"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("attribute", "unsafe_value"),
|
||||||
|
[
|
||||||
|
("role", "unexpected-owner"),
|
||||||
|
("can_login", True),
|
||||||
|
("is_superuser", True),
|
||||||
|
("can_create_db", True),
|
||||||
|
("can_create_role", True),
|
||||||
|
("inherits_privileges", True),
|
||||||
|
("can_replicate", True),
|
||||||
|
("bypasses_rls", True),
|
||||||
|
("connection_limit", 0),
|
||||||
|
("direct_membership_edges", 1),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_any_unsafe_stage_owner_role_posture_fails_closed(attribute: str, unsafe_value: object) -> None:
|
||||||
|
runner = FakeRunner(stage_owner_role_posture_override={attribute: unsafe_value})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "stage_owner_role_posture_mismatch"
|
||||||
|
assert caught.value.check == "stage_owner_role_posture"
|
||||||
|
assert SECRET_VALUE not in str(caught.value)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"function_name",
|
||||||
|
["approve_strict_proposal", "assert_approved_proposal", "finish_approved_proposal"],
|
||||||
|
)
|
||||||
|
def test_reviewer_or_apply_execute_grant_fails_closed(function_name: str) -> None:
|
||||||
|
runner = FakeRunner(function_posture_override={function_name: {"execute": True}})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "function_privilege_posture_mismatch"
|
||||||
|
assert caught.value.check == "function_privilege_posture"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"function_name",
|
||||||
|
[
|
||||||
|
"stage_leoclean_proposal_5_arg",
|
||||||
|
"approve_strict_proposal",
|
||||||
|
"assert_approved_proposal",
|
||||||
|
"finish_approved_proposal",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_missing_expected_exact_function_fails_closed(function_name: str) -> None:
|
||||||
|
runner = FakeRunner(function_posture_override={function_name: {"execute": False, "exists": False}})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "function_privilege_posture_mismatch"
|
||||||
|
assert caught.value.check == "function_privilege_posture"
|
||||||
|
|
||||||
|
|
||||||
|
def test_forged_overload_presence_or_missing_stage_execute_fails_closed() -> None:
|
||||||
|
forged = FakeRunner(function_posture_override={"stage_leoclean_proposal_6_arg": {"exists": True}})
|
||||||
|
missing_execute = FakeRunner(function_posture_override={"stage_leoclean_proposal_5_arg": {"execute": False}})
|
||||||
|
|
||||||
|
for runner in (forged, missing_execute):
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
assert caught.value.code == "function_privilege_posture_mismatch"
|
||||||
|
|
||||||
|
|
||||||
|
def _drifted_value(expected: object) -> object:
|
||||||
|
if isinstance(expected, bool):
|
||||||
|
return not expected
|
||||||
|
if isinstance(expected, int):
|
||||||
|
return expected + 1
|
||||||
|
if isinstance(expected, str):
|
||||||
|
return f"{expected}-drift"
|
||||||
|
if isinstance(expected, list):
|
||||||
|
return [*expected, "drift"]
|
||||||
|
raise AssertionError(f"unsupported test expectation type: {type(expected).__name__}")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("field", "unsafe_value"),
|
||||||
|
[
|
||||||
|
(field, _drifted_value(expected))
|
||||||
|
for field, expected in verifier.STAGE_FUNCTION_METADATA_EXPECTATIONS
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_any_stage_function_metadata_drift_fails_closed(field: str, unsafe_value: object) -> None:
|
||||||
|
runner = FakeRunner(stage_definition_override={field: unsafe_value})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "stage_function_definition_mismatch"
|
||||||
|
assert caught.value.check == "stage_function_definition"
|
||||||
|
assert SECRET_VALUE not in str(caught.value)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("field", "unsafe_value"),
|
||||||
|
[
|
||||||
|
("exists", False),
|
||||||
|
("argument_modes", ["i", "i", "i", "i", "i"]),
|
||||||
|
("configuration", ["search_path=public"]),
|
||||||
|
("acl_exact", False),
|
||||||
|
("runtime_execute", False),
|
||||||
|
("leakproof", 0),
|
||||||
|
("argument_defaults", False),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_any_stage_function_contract_drift_fails_closed(field: str, unsafe_value: object) -> None:
|
||||||
|
runner = FakeRunner(stage_definition_override={field: unsafe_value})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "stage_function_definition_mismatch"
|
||||||
|
assert caught.value.check == "stage_function_definition"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"unsafe_source",
|
||||||
|
[
|
||||||
|
None,
|
||||||
|
42,
|
||||||
|
verifier.EXPECTED_STAGE_FUNCTION_SOURCE.replace("return to_jsonb(staged);", "return '{}'::jsonb;"),
|
||||||
|
"x" * (verifier.MAX_STAGE_FUNCTION_SOURCE_BYTES + 1),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_missing_malformed_oversized_or_semantically_drifted_stage_body_fails_closed(
|
||||||
|
unsafe_source: object,
|
||||||
|
) -> None:
|
||||||
|
runner = FakeRunner(stage_definition_override={"source": unsafe_source})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "stage_function_definition_mismatch"
|
||||||
|
assert caught.value.check == "stage_function_definition"
|
||||||
|
failure = verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value))
|
||||||
|
assert "stage_leoclean_proposal is restricted" not in failure
|
||||||
|
assert SECRET_VALUE not in failure
|
||||||
|
|
||||||
|
|
||||||
|
def test_stage_body_normalization_accepts_only_line_ending_and_framing_newline_changes() -> None:
|
||||||
|
crlf_source = "\r\n" + verifier.EXPECTED_STAGE_FUNCTION_SOURCE.strip("\n").replace("\n", "\r\n") + "\r\n"
|
||||||
|
|
||||||
|
receipt = run_success(FakeRunner(stage_definition_override={"source": crlf_source}))
|
||||||
|
|
||||||
|
assert (
|
||||||
|
receipt["checks"]["stage_function_definition"]["source_sha256"]
|
||||||
|
== verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_embedded_stage_body_attestation_is_tied_to_the_provisioning_sql_source() -> None:
|
||||||
|
provisioning_sql = (
|
||||||
|
Path(__file__).resolve().parents[1] / "ops" / "gcp_leoclean_runtime_role.sql"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
function_anchor = "create or replace function kb_stage.stage_leoclean_proposal("
|
||||||
|
assert provisioning_sql.count(function_anchor) == 1
|
||||||
|
function_section = provisioning_sql.split(function_anchor, 1)[1]
|
||||||
|
assert function_section.count("as $function$") == 1
|
||||||
|
provisioned_source = function_section.split("as $function$", 1)[1].split("$function$;", 1)[0]
|
||||||
|
|
||||||
|
assert provisioned_source == verifier.EXPECTED_STAGE_FUNCTION_SOURCE
|
||||||
|
assert (
|
||||||
|
hashlib.sha256(verifier.normalize_stage_function_source(provisioned_source).encode("utf-8")).hexdigest()
|
||||||
|
== verifier.EXPECTED_STAGE_FUNCTION_SOURCE_SHA256
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_catalog_queries_use_exact_regprocedure_signatures_and_both_membership_directions() -> None:
|
||||||
|
function_sql = verifier._function_privilege_posture_sql()
|
||||||
|
role_sql = verifier._role_posture_sql()
|
||||||
|
owner_role_sql = verifier._stage_owner_role_posture_sql()
|
||||||
|
stage_definition_sql = verifier._stage_function_definition_sql()
|
||||||
|
catalog_sql = verifier._catalog_privilege_posture_sql()
|
||||||
|
legacy_sql = verifier._legacy_catalog_privilege_posture_sql()
|
||||||
|
|
||||||
|
assert "pg_catalog.to_regprocedure(signature)" in function_sql
|
||||||
|
for _name, signature, _exists, _execute in verifier.FUNCTION_PRIVILEGE_EXPECTATIONS:
|
||||||
|
assert signature in function_sql
|
||||||
|
assert "membership.roleid = role_row.oid" in role_sql
|
||||||
|
assert "membership.member = role_row.oid" in role_sql
|
||||||
|
assert "membership.roleid = role_row.oid" in owner_role_sql
|
||||||
|
assert "membership.member = role_row.oid" in owner_role_sql
|
||||||
|
assert verifier.STAGE_OWNER_DATABASE_ROLE in owner_role_sql
|
||||||
|
assert verifier.STAGE_FUNCTION_SIGNATURE in stage_definition_sql
|
||||||
|
assert "left join pg_catalog.pg_proc" in stage_definition_sql
|
||||||
|
assert "function_catalog.prosrc" in stage_definition_sql
|
||||||
|
assert "function_catalog.prokind" in stage_definition_sql
|
||||||
|
assert "function_catalog.proargtypes" in stage_definition_sql
|
||||||
|
assert "function_catalog.prosecdef" in stage_definition_sql
|
||||||
|
assert "pg_catalog.pg_get_function_result" in stage_definition_sql
|
||||||
|
assert "pg_catalog.aclexplode" in stage_definition_sql
|
||||||
|
assert "pg_catalog.pg_shdepend" in catalog_sql
|
||||||
|
assert "pg_catalog.has_table_privilege" in catalog_sql
|
||||||
|
assert "pg_catalog.has_column_privilege" in catalog_sql
|
||||||
|
assert "pg_catalog.has_sequence_privilege" in catalog_sql
|
||||||
|
assert "pg_catalog.has_function_privilege" in catalog_sql
|
||||||
|
assert "pg_catalog.aclexplode" in catalog_sql
|
||||||
|
assert "nspname !~ '^pg_'" in catalog_sql
|
||||||
|
assert "nspname <> 'information_schema'" in catalog_sql
|
||||||
|
assert verifier.STAGE_OWNER_DATABASE_ROLE in catalog_sql
|
||||||
|
assert "teleo_restore" in legacy_sql
|
||||||
|
assert "pg_catalog.has_schema_privilege" in legacy_sql
|
||||||
|
assert "pg_catalog.has_table_privilege" in legacy_sql
|
||||||
|
assert "pg_catalog.has_column_privilege" in legacy_sql
|
||||||
|
assert "pg_catalog.has_sequence_privilege" in legacy_sql
|
||||||
|
assert "pg_catalog.has_function_privilege" in legacy_sql
|
||||||
|
for schema, relation in verifier.READ_TABLE_ALLOWLIST:
|
||||||
|
assert schema in catalog_sql
|
||||||
|
assert relation in catalog_sql
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("field", verifier.CATALOG_ZERO_COUNT_FIELDS)
|
||||||
|
def test_any_unexpected_catalog_privilege_fails_closed(field: str) -> None:
|
||||||
|
runner = FakeRunner(catalog_posture_override={field: 1})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "catalog_privilege_posture_mismatch"
|
||||||
|
assert caught.value.check == "catalog_privilege_posture"
|
||||||
|
assert SECRET_VALUE not in str(caught.value)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("field", verifier.CATALOG_TRUE_FIELDS)
|
||||||
|
def test_any_missing_required_catalog_privilege_fails_closed(field: str) -> None:
|
||||||
|
runner = FakeRunner(catalog_posture_override={field: False})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "catalog_privilege_posture_mismatch"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("field", verifier.LEGACY_CATALOG_ZERO_COUNT_FIELDS)
|
||||||
|
def test_any_teleo_restore_catalog_privilege_fails_closed(field: str) -> None:
|
||||||
|
runner = FakeRunner(legacy_catalog_posture_override={field: 1})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "legacy_catalog_privilege_posture_mismatch"
|
||||||
|
assert caught.value.check == "legacy_catalog_privileges"
|
||||||
|
assert SECRET_VALUE not in str(caught.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_teleo_restore_catalog_and_behavioral_denial_are_both_exercised_on_legacy_database() -> None:
|
||||||
|
runner = FakeRunner()
|
||||||
|
|
||||||
|
receipt = run_success(runner)
|
||||||
|
|
||||||
|
legacy_catalog_call = next(
|
||||||
|
command
|
||||||
|
for command, _env, _timeout, _discard in runner.calls
|
||||||
|
if any(part == f"--command={verifier._legacy_catalog_privilege_posture_sql()}" for part in command)
|
||||||
|
)
|
||||||
|
assert any(f"dbname={verifier.LEGACY_DATABASE}" in part for part in legacy_catalog_call)
|
||||||
|
legacy_denial = next(
|
||||||
|
check for check in receipt["checks"]["negative_permissions"] if check["check"] == "legacy_teleo_restore_read"
|
||||||
|
)
|
||||||
|
assert legacy_denial == {
|
||||||
|
"check": "legacy_teleo_restore_read",
|
||||||
|
"expected_sqlstate": "42501",
|
||||||
|
"observed_sqlstate": "42501",
|
||||||
|
"result": "denied",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_required_sqlstate_mismatch_fails_closed_without_secret_in_error() -> None:
|
||||||
|
runner = FakeRunner(sqlstate_override=("canonical_insert", "23505"))
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
error = caught.value
|
||||||
|
assert error.code == "negative_permission_sqlstate_mismatch"
|
||||||
|
assert error.check == "canonical_insert"
|
||||||
|
assert error.expected_sqlstate == "42501"
|
||||||
|
assert error.observed_sqlstate == "23505"
|
||||||
|
assert SECRET_VALUE not in str(error)
|
||||||
|
assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, error))
|
||||||
|
assert_child_boundaries(runner)
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_verbose_sqlstate_fails_closed_without_raw_stderr() -> None:
|
||||||
|
runner = FakeRunner(sqlstate_missing_for="forged_proposer_overload")
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
error = caught.value
|
||||||
|
assert error.code == "negative_permission_sqlstate_missing"
|
||||||
|
assert error.expected_sqlstate == "42883"
|
||||||
|
assert error.observed_sqlstate == "missing"
|
||||||
|
assert SECRET_VALUE not in json.dumps(error.as_dict())
|
||||||
|
|
||||||
|
|
||||||
|
def test_unexpected_permission_success_fails_closed_and_transaction_sql_has_rollback() -> None:
|
||||||
|
runner = FakeRunner(unexpected_success_for="proposal_update")
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "negative_permission_unexpectedly_succeeded"
|
||||||
|
assert caught.value.observed_sqlstate == "success"
|
||||||
|
proposal_update = next(
|
||||||
|
check for check in verifier._negative_checks(RUN_ID, SOURCE_REF) if check.name == "proposal_update"
|
||||||
|
)
|
||||||
|
assert proposal_update.sql.startswith("begin;\n")
|
||||||
|
assert proposal_update.sql.endswith("\nrollback;")
|
||||||
|
assert SECRET_VALUE not in str(caught.value)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("administrator_mode", ["success", "wrong-error"])
|
||||||
|
def test_administrator_secret_must_be_exact_iam_denial_and_stdout_is_discarded(administrator_mode: str) -> None:
|
||||||
|
runner = FakeRunner(administrator_mode=administrator_mode)
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
expected_code = (
|
||||||
|
"administrator_secret_unexpectedly_readable"
|
||||||
|
if administrator_mode == "success"
|
||||||
|
else "administrator_secret_denial_not_iam_permission"
|
||||||
|
)
|
||||||
|
assert caught.value.code == expected_code
|
||||||
|
assert SECRET_VALUE not in str(caught.value)
|
||||||
|
assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value))
|
||||||
|
assert runner.calls[-1][3] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_scoped_secret_failure_never_repeats_command_output() -> None:
|
||||||
|
runner = FakeRunner(scoped_failure=True)
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "scoped_secret_access_failed"
|
||||||
|
assert SECRET_VALUE not in str(caught.value)
|
||||||
|
assert SECRET_VALUE not in verifier.canonical_json(verifier.failure_receipt(RUN_ID, caught.value))
|
||||||
|
assert len(runner.calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"invalid",
|
||||||
|
["short", "UPPERCASE-RUN", "contains_underscore", "contains.dot", " leading-space", "a" * 65],
|
||||||
|
)
|
||||||
|
def test_run_id_validation_is_strict(invalid: str) -> None:
|
||||||
|
with pytest.raises(ValueError, match="8-64 lowercase"):
|
||||||
|
verifier.validate_run_id(invalid)
|
||||||
|
|
||||||
|
|
||||||
|
def test_wrong_identity_or_non_ssl_connection_fails_before_permission_probes() -> None:
|
||||||
|
runner = FakeRunner(identity_override={"ssl": False})
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
run_success(runner)
|
||||||
|
|
||||||
|
assert caught.value.code == "private_ssl_identity_mismatch"
|
||||||
|
assert len(runner.calls) == 2
|
||||||
|
assert SECRET_VALUE not in str(caught.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_optional_receipt_is_canonical_mode_0600_and_refuses_symlink(tmp_path: Path) -> None:
|
||||||
|
payload = {"z": 2, "a": {"safe": True}}
|
||||||
|
output = tmp_path / "receipt.json"
|
||||||
|
|
||||||
|
verifier.write_receipt(output, payload)
|
||||||
|
|
||||||
|
assert output.read_text(encoding="utf-8") == verifier.canonical_json(payload)
|
||||||
|
assert stat.S_IMODE(output.stat().st_mode) == 0o600
|
||||||
|
assert output.read_text(encoding="utf-8").startswith('{"a":')
|
||||||
|
|
||||||
|
target = tmp_path / "target.json"
|
||||||
|
target.write_text("untouched", encoding="utf-8")
|
||||||
|
symlink = tmp_path / "symlink.json"
|
||||||
|
symlink.symlink_to(target)
|
||||||
|
with pytest.raises(OSError):
|
||||||
|
verifier.write_receipt(symlink, payload)
|
||||||
|
assert target.read_text(encoding="utf-8") == "untouched"
|
||||||
|
|
||||||
|
|
||||||
|
def test_subprocess_runner_discards_administrator_stdout_at_file_descriptor(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
observed: dict[str, object] = {}
|
||||||
|
|
||||||
|
def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
|
||||||
|
observed["command"] = command
|
||||||
|
observed.update(kwargs)
|
||||||
|
return subprocess.CompletedProcess(command, 1, stdout=None, stderr="PERMISSION_DENIED")
|
||||||
|
|
||||||
|
monkeypatch.setattr(verifier.subprocess, "run", fake_run)
|
||||||
|
|
||||||
|
result = verifier.subprocess_runner([verifier.GCLOUD_BIN, "test"], os.environ, 3, True)
|
||||||
|
|
||||||
|
assert observed["stdout"] is subprocess.DEVNULL
|
||||||
|
assert observed["stdin"] is subprocess.DEVNULL
|
||||||
|
assert observed["stderr"] is subprocess.PIPE
|
||||||
|
assert result.stdout == ""
|
||||||
|
assert result.returncode == 1
|
||||||
281
tests/test_gcp_leoclean_service_environment.py
Normal file
281
tests/test_gcp_leoclean_service_environment.py
Normal file
|
|
@ -0,0 +1,281 @@
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from ops import verify_gcp_leoclean_service_environment as verifier
|
||||||
|
|
||||||
|
|
||||||
|
def environment_block(environment: Mapping[str, str]) -> bytes:
|
||||||
|
return b"\0".join(f"{key}={value}".encode() for key, value in environment.items()) + b"\0"
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_expected_fields_pass_and_receipt_contains_no_values() -> None:
|
||||||
|
environment = verifier.expected_environment()
|
||||||
|
environment.update({"HOME": "/home/teleo", "LANG": "C.UTF-8"})
|
||||||
|
|
||||||
|
parsed = verifier.parse_environment(environment_block(environment))
|
||||||
|
validated = verifier.validate_environment(parsed)
|
||||||
|
receipt = {
|
||||||
|
"runtime_environment": "scoped",
|
||||||
|
"status": "pass",
|
||||||
|
"validated_fields": list(validated),
|
||||||
|
}
|
||||||
|
rendered = verifier.canonical_json(receipt)
|
||||||
|
|
||||||
|
assert validated == tuple(sorted(verifier.expected_environment()))
|
||||||
|
assert json.loads(rendered) == receipt
|
||||||
|
assert rendered == json.dumps(receipt, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + "\n"
|
||||||
|
for value in verifier.expected_environment().values():
|
||||||
|
assert value not in rendered
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("field", sorted(verifier.expected_environment()))
|
||||||
|
def test_each_expected_field_mismatch_fails_closed(field: str) -> None:
|
||||||
|
environment = verifier.expected_environment()
|
||||||
|
environment[field] = "wrong-value-that-is-not-a-secret"
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
verifier.validate_environment(environment)
|
||||||
|
|
||||||
|
assert caught.value.code == "environment_mismatch"
|
||||||
|
assert caught.value.fields == (field,)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("field", "unsafe_value", "expected_code"),
|
||||||
|
[
|
||||||
|
("TELEO_CLOUDSQL_USER", "postgres", "environment_mismatch"),
|
||||||
|
(
|
||||||
|
"TELEO_CLOUDSQL_PASSWORD_SECRET",
|
||||||
|
verifier.ADMINISTRATOR_PASSWORD_SECRET,
|
||||||
|
"administrator_secret_reference",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_later_dropin_effective_override_fails_without_outputting_value(
|
||||||
|
field: str,
|
||||||
|
unsafe_value: str,
|
||||||
|
expected_code: str,
|
||||||
|
) -> None:
|
||||||
|
environment = verifier.expected_environment()
|
||||||
|
environment[field] = unsafe_value
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
verifier.validate_environment(environment)
|
||||||
|
|
||||||
|
rendered = verifier.canonical_json(verifier.failure_receipt(caught.value))
|
||||||
|
assert caught.value.code == expected_code
|
||||||
|
assert unsafe_value not in rendered
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("field", "safe_label"),
|
||||||
|
[
|
||||||
|
("PGPASSWORD", "PG*"),
|
||||||
|
("pgservice", "PG*"),
|
||||||
|
("CLOUDSDK_CONFIG", "CLOUDSDK_*"),
|
||||||
|
("CLOUDSDK_CORE_PROJECT", "CLOUDSDK_*"),
|
||||||
|
("CLOUDSDK_AUTH_ACCESS_TOKEN", "CLOUDSDK_*"),
|
||||||
|
("cloudsdk_auth_refresh_token", "CLOUDSDK_*"),
|
||||||
|
("GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_APPLICATION_CREDENTIALS"),
|
||||||
|
("LD_PRELOAD", "LD_*"),
|
||||||
|
("LD_LIBRARY_PATH", "LD_*"),
|
||||||
|
("ld_audit", "LD_*"),
|
||||||
|
("BASH_ENV", "BASH_ENV"),
|
||||||
|
("ENV", "ENV"),
|
||||||
|
("BASH_FUNC_injected%%", "BASH_FUNC_*"),
|
||||||
|
("PYTHONHOME", "PYTHON*"),
|
||||||
|
("PYTHONPATH", "PYTHON*"),
|
||||||
|
("PYTHONHTTPSVERIFY", "PYTHON*"),
|
||||||
|
("pythonstartup", "PYTHON*"),
|
||||||
|
("HTTP_PROXY", "HTTP_PROXY"),
|
||||||
|
("https_proxy", "HTTPS_PROXY"),
|
||||||
|
("ALL_PROXY", "ALL_PROXY"),
|
||||||
|
("no_proxy", "NO_PROXY"),
|
||||||
|
("SSL_CERT_FILE", "SSL_CERT_FILE"),
|
||||||
|
("ssl_cert_dir", "SSL_CERT_DIR"),
|
||||||
|
("REQUESTS_CA_BUNDLE", "REQUESTS_CA_BUNDLE"),
|
||||||
|
("curl_ca_bundle", "CURL_CA_BUNDLE"),
|
||||||
|
("SSLKEYLOGFILE", "SSLKEYLOGFILE"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_environment_file_broader_credentials_and_forbidden_fields_fail_closed(
|
||||||
|
field: str,
|
||||||
|
safe_label: str,
|
||||||
|
) -> None:
|
||||||
|
environment = verifier.expected_environment()
|
||||||
|
injected_value = "sensitive-injected-value"
|
||||||
|
environment[field] = injected_value
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
verifier.validate_environment(environment)
|
||||||
|
|
||||||
|
rendered = verifier.canonical_json(verifier.failure_receipt(caught.value))
|
||||||
|
assert caught.value.code == "forbidden_environment_fields"
|
||||||
|
assert caught.value.fields == (safe_label,)
|
||||||
|
assert injected_value not in rendered
|
||||||
|
|
||||||
|
|
||||||
|
def test_arbitrary_forbidden_key_suffix_is_not_an_output_channel() -> None:
|
||||||
|
environment = verifier.expected_environment()
|
||||||
|
environment["PG_SECRET_MARKER"] = "not-output"
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
verifier.validate_environment(environment)
|
||||||
|
|
||||||
|
rendered = verifier.canonical_json(verifier.failure_receipt(caught.value))
|
||||||
|
assert "PG_SECRET_MARKER" not in rendered
|
||||||
|
assert "PG*" in rendered
|
||||||
|
|
||||||
|
|
||||||
|
def test_pythonunbuffered_is_the_only_allowed_python_runtime_tuning() -> None:
|
||||||
|
environment = verifier.expected_environment()
|
||||||
|
environment["PYTHONUNBUFFERED"] = "1"
|
||||||
|
|
||||||
|
assert verifier.validate_environment(environment) == tuple(sorted(verifier.expected_environment()))
|
||||||
|
|
||||||
|
|
||||||
|
def test_bash_env_executes_before_a_script_but_effective_verifier_rejects_it(tmp_path: Path) -> None:
|
||||||
|
marker = "BASH_ENV_EXECUTED"
|
||||||
|
payload = tmp_path / "bash-env.sh"
|
||||||
|
payload.write_text(f"printf '{marker}\\n'\n", encoding="utf-8")
|
||||||
|
completed = subprocess.run(
|
||||||
|
["/bin/bash", "-c", ":"],
|
||||||
|
env={"BASH_ENV": str(payload), "PATH": verifier.RUNTIME_PATH},
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert completed.returncode == 0
|
||||||
|
assert completed.stdout.strip() == marker
|
||||||
|
|
||||||
|
environment = verifier.expected_environment()
|
||||||
|
environment["BASH_ENV"] = str(payload)
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
verifier.validate_environment(environment)
|
||||||
|
assert caught.value.fields == ("BASH_ENV",)
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_secret_reference_in_any_value_fails_without_field_or_value_leak() -> None:
|
||||||
|
environment = verifier.expected_environment()
|
||||||
|
environment["UNRELATED"] = f"prefix/{verifier.ADMINISTRATOR_PASSWORD_SECRET}/suffix"
|
||||||
|
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
verifier.validate_environment(environment)
|
||||||
|
|
||||||
|
rendered = verifier.canonical_json(verifier.failure_receipt(caught.value))
|
||||||
|
assert caught.value.code == "administrator_secret_reference"
|
||||||
|
assert verifier.ADMINISTRATOR_PASSWORD_SECRET not in rendered
|
||||||
|
assert "UNRELATED" not in rendered
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"raw",
|
||||||
|
[
|
||||||
|
b"A=1",
|
||||||
|
b"A=1\0\0",
|
||||||
|
b"A=1\0BROKEN\0",
|
||||||
|
b"=value\0",
|
||||||
|
b"A=1\0A=2\0",
|
||||||
|
b"A=\xff\0",
|
||||||
|
b"",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_malformed_nul_environment_is_rejected(raw: bytes) -> None:
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
verifier.parse_environment(raw)
|
||||||
|
|
||||||
|
assert caught.value.code == "environment_malformed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_preserves_equals_inside_values() -> None:
|
||||||
|
assert verifier.parse_environment(b"A=one=two\0B=\0") == {"A": "one=two", "B": ""}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("value", [0, -1, 1.5, "0", "01", "+1", "-3", "not-a-pid", True])
|
||||||
|
def test_pid_must_be_a_positive_integer(value: object) -> None:
|
||||||
|
with pytest.raises(verifier.VerificationError) as caught:
|
||||||
|
verifier.validate_pid(value) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
assert caught.value.code == "invalid_pid"
|
||||||
|
|
||||||
|
|
||||||
|
def test_cli_errors_are_canonical_sanitized_json_and_exit_one(capsys: pytest.CaptureFixture[str]) -> None:
|
||||||
|
secret_input = "not-a-pid-secret-marker"
|
||||||
|
|
||||||
|
returncode = verifier.main(["--pid", secret_input])
|
||||||
|
|
||||||
|
output = capsys.readouterr()
|
||||||
|
assert returncode == 1
|
||||||
|
assert output.err == ""
|
||||||
|
assert json.loads(output.out) == {"error": {"code": "invalid_pid"}, "status": "fail"}
|
||||||
|
assert secret_input not in output.out
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="requires Linux /proc/PID/environ")
|
||||||
|
def test_cli_reads_real_process_environment_and_detects_effective_conflict(
|
||||||
|
capsys: pytest.CaptureFixture[str],
|
||||||
|
) -> None:
|
||||||
|
expected = verifier.expected_environment()
|
||||||
|
good_process = subprocess.Popen(
|
||||||
|
["/bin/sleep", "30"],
|
||||||
|
env=expected,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
assert verifier.main(["--pid", str(good_process.pid)]) == 0
|
||||||
|
good_output = capsys.readouterr()
|
||||||
|
assert good_output.err == ""
|
||||||
|
assert json.loads(good_output.out)["status"] == "pass"
|
||||||
|
finally:
|
||||||
|
good_process.terminate()
|
||||||
|
good_process.wait(timeout=5)
|
||||||
|
|
||||||
|
conflicting = dict(expected)
|
||||||
|
conflicting["TELEO_CLOUDSQL_USER"] = "postgres"
|
||||||
|
conflicting["GOOGLE_APPLICATION_CREDENTIALS"] = "/tmp/broader-credential.json"
|
||||||
|
bad_process = subprocess.Popen(
|
||||||
|
["/bin/sleep", "30"],
|
||||||
|
env=conflicting,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
assert verifier.main(["--pid", str(bad_process.pid)]) == 1
|
||||||
|
bad_output = capsys.readouterr()
|
||||||
|
receipt = json.loads(bad_output.out)
|
||||||
|
assert bad_output.err == ""
|
||||||
|
assert receipt == {
|
||||||
|
"error": {"code": "forbidden_environment_fields", "fields": ["GOOGLE_APPLICATION_CREDENTIALS"]},
|
||||||
|
"status": "fail",
|
||||||
|
}
|
||||||
|
assert "postgres" not in bad_output.out
|
||||||
|
assert "/tmp/broader-credential.json" not in bad_output.out
|
||||||
|
finally:
|
||||||
|
bad_process.terminate()
|
||||||
|
bad_process.wait(timeout=5)
|
||||||
|
|
||||||
|
|
||||||
|
def test_unexpected_read_error_is_generic_and_does_not_echo_path(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
capsys: pytest.CaptureFixture[str],
|
||||||
|
) -> None:
|
||||||
|
sensitive_marker = "sensitive-proc-read-detail"
|
||||||
|
|
||||||
|
def fail_read(_self: object) -> bytes:
|
||||||
|
raise OSError(sensitive_marker)
|
||||||
|
|
||||||
|
monkeypatch.setattr(verifier.Path, "read_bytes", fail_read)
|
||||||
|
|
||||||
|
assert verifier.main(["--pid", str(os.getpid())]) == 1
|
||||||
|
output = capsys.readouterr()
|
||||||
|
assert json.loads(output.out) == {"error": {"code": "internal_error"}, "status": "fail"}
|
||||||
|
assert sensitive_marker not in output.out
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import ast
|
import ast
|
||||||
|
import base64
|
||||||
import contextlib
|
import contextlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import io
|
import io
|
||||||
|
|
@ -64,6 +65,14 @@ def test_cloudsql_wrapper_supports_expected_review_gated_commands() -> None:
|
||||||
assert command in cloudsql_text
|
assert command in cloudsql_text
|
||||||
|
|
||||||
assert "status" in vps_text
|
assert "status" in vps_text
|
||||||
|
assert "SCRIPT_DIR=${SCRIPT_PATH%/*}" in wrapper_text
|
||||||
|
assert 'SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"' in wrapper_text
|
||||||
|
assert 'CLOUDSQL_TOOL="$SCRIPT_DIR/cloudsql_memory_tool.py"' in wrapper_text
|
||||||
|
assert wrapper_text.startswith("#!/bin/bash\n")
|
||||||
|
assert "exec /usr/bin/python3" in wrapper_text
|
||||||
|
assert "command -v psql" in wrapper_text
|
||||||
|
assert "gcloud" not in wrapper_text
|
||||||
|
assert '"gcloud"' not in cloudsql_text
|
||||||
|
|
||||||
assert "kb_stage.kb_proposals" in cloudsql_text
|
assert "kb_stage.kb_proposals" in cloudsql_text
|
||||||
assert "canonical apply" in cloudsql_text.lower()
|
assert "canonical apply" in cloudsql_text.lower()
|
||||||
|
|
@ -95,7 +104,7 @@ def _install_wrapper_fixture(tmp_path: Path, *, missing: frozenset[str] = frozen
|
||||||
target = shutil.which(name)
|
target = shutil.which(name)
|
||||||
assert target
|
assert target
|
||||||
(fake_bin / name).symlink_to(target)
|
(fake_bin / name).symlink_to(target)
|
||||||
for name in ("psql", "gcloud"):
|
for name in ("psql",):
|
||||||
if name in missing:
|
if name in missing:
|
||||||
continue
|
continue
|
||||||
executable = fake_bin / name
|
executable = fake_bin / name
|
||||||
|
|
@ -146,7 +155,7 @@ def test_cloudsql_wrapper_auto_mode_never_falls_back_after_supported_command_fai
|
||||||
assert "local" not in completed.stdout
|
assert "local" not in completed.stdout
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("missing", ["cloudsql-tool", "psql", "gcloud"])
|
@pytest.mark.parametrize("missing", ["cloudsql-tool", "psql"])
|
||||||
def test_cloudsql_wrapper_auto_mode_fails_closed_when_prerequisite_is_missing(tmp_path: Path, missing: str) -> None:
|
def test_cloudsql_wrapper_auto_mode_fails_closed_when_prerequisite_is_missing(tmp_path: Path, missing: str) -> None:
|
||||||
wrapper, env = _install_wrapper_fixture(tmp_path, missing=frozenset({missing}))
|
wrapper, env = _install_wrapper_fixture(tmp_path, missing=frozenset({missing}))
|
||||||
|
|
||||||
|
|
@ -209,6 +218,72 @@ def test_bridge_source_does_not_commit_raw_secret_values() -> None:
|
||||||
assert not re.search(pattern, combined), pattern
|
assert not re.search(pattern, combined), pattern
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime_connection_args(module, **overrides):
|
||||||
|
values = {
|
||||||
|
"credential_mode": "runtime",
|
||||||
|
"user": module.RUNTIME_DB_USER,
|
||||||
|
"password_secret": module.RUNTIME_PASSWORD_SECRET,
|
||||||
|
"project": module.RUNTIME_GCP_PROJECT,
|
||||||
|
"host": module.RUNTIME_CLOUDSQL_HOST,
|
||||||
|
"port": module.RUNTIME_CLOUDSQL_PORT,
|
||||||
|
"db": module.RUNTIME_CLOUDSQL_DB,
|
||||||
|
"canonical_db": module.RUNTIME_CLOUDSQL_DB,
|
||||||
|
"command": "status",
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return SimpleNamespace(**values)
|
||||||
|
|
||||||
|
|
||||||
|
def _configure_runtime_environment(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
for key in tuple(os.environ):
|
||||||
|
normalized_key = key.upper()
|
||||||
|
if (
|
||||||
|
normalized_key.startswith("PG")
|
||||||
|
or normalized_key.startswith("CLOUDSDK_")
|
||||||
|
or normalized_key == "GOOGLE_APPLICATION_CREDENTIALS"
|
||||||
|
or normalized_key in {"LD_PRELOAD", "PYTHONHOME", "PYTHONPATH"}
|
||||||
|
or normalized_key == "PROXY"
|
||||||
|
or normalized_key.endswith("_PROXY")
|
||||||
|
or normalized_key in {
|
||||||
|
"CURL_CA_BUNDLE",
|
||||||
|
"PYTHONHTTPSVERIFY",
|
||||||
|
"REQUESTS_CA_BUNDLE",
|
||||||
|
"SSL_CERT_DIR",
|
||||||
|
"SSL_CERT_FILE",
|
||||||
|
"SSLKEYLOGFILE",
|
||||||
|
}
|
||||||
|
):
|
||||||
|
monkeypatch.delenv(key, raising=False)
|
||||||
|
monkeypatch.setenv("TELEO_GCP_METADATA_ONLY", "1")
|
||||||
|
monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0")
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeHttpResponse:
|
||||||
|
def __init__(self, body: bytes, status: int = 200) -> None:
|
||||||
|
self.body = body
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def getcode(self) -> int:
|
||||||
|
return self.status
|
||||||
|
|
||||||
|
def read(self, _limit: int) -> bytes:
|
||||||
|
return self.body
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime_http_bodies(module, password: bytes = b"scoped-runtime-password") -> list[bytes]:
|
||||||
|
return [
|
||||||
|
module.RUNTIME_SERVICE_ACCOUNT.encode("ascii"),
|
||||||
|
json.dumps({"access_token": "metadata-access-token", "token_type": "Bearer", "expires_in": 300}).encode(),
|
||||||
|
json.dumps({"payload": {"data": base64.b64encode(password).decode("ascii")}}).encode(),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_cloudsql_runtime_requires_the_exact_scoped_identity(monkeypatch: pytest.MonkeyPatch) -> None:
|
def test_cloudsql_runtime_requires_the_exact_scoped_identity(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
monkeypatch.delenv("PGPASSWORD", raising=False)
|
monkeypatch.delenv("PGPASSWORD", raising=False)
|
||||||
|
|
@ -249,17 +324,113 @@ def test_cloudsql_runtime_accepts_only_scoped_credentials_and_rejects_inherited_
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
args = SimpleNamespace(
|
args = _runtime_connection_args(module)
|
||||||
credential_mode="runtime",
|
_configure_runtime_environment(monkeypatch)
|
||||||
user=module.RUNTIME_DB_USER,
|
|
||||||
password_secret=module.RUNTIME_PASSWORD_SECRET,
|
|
||||||
)
|
|
||||||
monkeypatch.delenv("PGPASSWORD", raising=False)
|
|
||||||
|
|
||||||
module.validate_runtime_connection(args)
|
module.validate_runtime_connection(args)
|
||||||
|
|
||||||
monkeypatch.setenv("PGPASSWORD", "")
|
monkeypatch.setenv("PGPASSWORD", "")
|
||||||
with pytest.raises(SystemExit, match="Refusing inherited PGPASSWORD"):
|
with pytest.raises(SystemExit, match="Refusing inherited credential, proxy, TLS, or PostgreSQL environment"):
|
||||||
|
module.validate_runtime_connection(args)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("name", "value"),
|
||||||
|
[
|
||||||
|
("GOOGLE_APPLICATION_CREDENTIALS", "/tmp/ambient.json"),
|
||||||
|
("google_application_credentials", "/tmp/ambient-lowercase.json"),
|
||||||
|
("CLOUDSDK_CONFIG", "/tmp/gcloud"),
|
||||||
|
("CLOUDSDK_AUTH_ACCESS_TOKEN", "ambient-token"),
|
||||||
|
("cloudsdk_core_project", "attacker-project"),
|
||||||
|
("cloudsdk_auth_access_token", "ambient-lowercase-token"),
|
||||||
|
("PGHOST", "attacker.invalid"),
|
||||||
|
("PGOPTIONS", "-c search_path=attacker"),
|
||||||
|
("pgservice", "attacker-service"),
|
||||||
|
("ld_preload", "/tmp/attacker.so"),
|
||||||
|
("LD_LIBRARY_PATH", "/tmp/attacker-library"),
|
||||||
|
("Ld_AuDiT", "/tmp/attacker-audit.so"),
|
||||||
|
("BASH_ENV", "/tmp/attacker-bash-env"),
|
||||||
|
("ENV", "/tmp/attacker-sh-env"),
|
||||||
|
("BASH_FUNC_psql%%", "() { :; }"),
|
||||||
|
("PythonHome", "/tmp/attacker-python"),
|
||||||
|
("pythonpath", "/tmp/attacker-modules"),
|
||||||
|
("PYTHONHTTPSVERIFY", "0"),
|
||||||
|
("pythonstartup", "/tmp/attacker-startup.py"),
|
||||||
|
("https_proxy", "http://attacker.invalid:8080"),
|
||||||
|
("HtTp_PrOxY", "http://attacker.invalid:8080"),
|
||||||
|
("ssl_cert_file", "/tmp/attacker.pem"),
|
||||||
|
("SSL_CERT_DIR", "/tmp/attacker-certs"),
|
||||||
|
("REQUESTS_CA_BUNDLE", "/tmp/attacker.pem"),
|
||||||
|
("SSLKEYLOGFILE", "/tmp/attacker-tls-keys"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_cloudsql_runtime_rejects_ambient_auth_and_pg_environment(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
name: str,
|
||||||
|
value: str,
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
_configure_runtime_environment(monkeypatch)
|
||||||
|
monkeypatch.setenv(name, value)
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit, match="Refusing inherited credential, proxy, TLS, or PostgreSQL environment"):
|
||||||
|
module.validate_runtime_connection(args)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cloudsql_runtime_allows_only_pythonunbuffered_runtime_tuning(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
_configure_runtime_environment(monkeypatch)
|
||||||
|
monkeypatch.setenv("PYTHONUNBUFFERED", "1")
|
||||||
|
|
||||||
|
module.validate_runtime_connection(args)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cloudsql_runtime_requires_explicit_metadata_only_mode(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
_configure_runtime_environment(monkeypatch)
|
||||||
|
monkeypatch.delenv("TELEO_GCP_METADATA_ONLY")
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit, match="TELEO_GCP_METADATA_ONLY=1"):
|
||||||
|
module.validate_runtime_connection(args)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("field", "value", "message"),
|
||||||
|
[
|
||||||
|
("project", "other-project", "project=teleo-501523"),
|
||||||
|
("host", "127.0.0.1", "host=10.61.0.3"),
|
||||||
|
("port", "6543", "port=5432"),
|
||||||
|
("db", "teleo_kb", "db=teleo_canonical"),
|
||||||
|
("canonical_db", "teleo_kb", "canonical_db=teleo_canonical"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_cloudsql_runtime_rejects_any_target_drift_before_credentials(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
field: str,
|
||||||
|
value: str,
|
||||||
|
message: str,
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module, **{field: value})
|
||||||
|
monkeypatch.delenv("PGPASSWORD", raising=False)
|
||||||
|
monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "0")
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit, match=message):
|
||||||
|
module.validate_runtime_connection(args)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cloudsql_runtime_rejects_audit_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
monkeypatch.delenv("PGPASSWORD", raising=False)
|
||||||
|
monkeypatch.setenv("TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK", "1")
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit, match="AUDIT_FALLBACK=0"):
|
||||||
module.validate_runtime_connection(args)
|
module.validate_runtime_connection(args)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -290,33 +461,44 @@ def test_cloudsql_clone_mode_is_read_only_and_bound_to_one_disposable_database(
|
||||||
module.validate_runtime_connection(args)
|
module.validate_runtime_connection(args)
|
||||||
|
|
||||||
|
|
||||||
def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pytest.MonkeyPatch) -> None:
|
def test_cloudsql_clone_mode_never_resolves_a_password_secret_argument(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
args = SimpleNamespace(
|
args = SimpleNamespace(
|
||||||
credential_mode="runtime",
|
credential_mode="clone-readonly",
|
||||||
user=module.RUNTIME_DB_USER,
|
user="postgres",
|
||||||
password_secret=module.RUNTIME_PASSWORD_SECRET,
|
password_secret="gcp-teleo-pgvector-standby-postgres-password",
|
||||||
host="10.61.0.3",
|
canonical_db="teleo_clone_canary_123",
|
||||||
port="5432",
|
db="teleo_clone_canary_123",
|
||||||
db="teleo_canonical",
|
|
||||||
canonical_db="teleo_canonical",
|
|
||||||
project="teleo-501523",
|
|
||||||
command="status",
|
command="status",
|
||||||
)
|
)
|
||||||
monkeypatch.delenv("PGPASSWORD", raising=False)
|
monkeypatch.delenv("PGPASSWORD", raising=False)
|
||||||
monkeypatch.setenv("PGHOST", "attacker.invalid")
|
monkeypatch.setenv("PGOPTIONS", module.CLONE_READ_ONLY_PGOPTIONS)
|
||||||
monkeypatch.setenv("PGSERVICE", "untrusted-service")
|
monkeypatch.setattr(
|
||||||
monkeypatch.setenv("PGPASSFILE", "/tmp/untrusted-pgpass")
|
module.subprocess,
|
||||||
monkeypatch.setenv("PGOPTIONS", "-c search_path=attacker")
|
"run",
|
||||||
|
lambda *_args, **_kwargs: pytest.fail("a password-secret argument must never launch a credential helper"),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit, match="requires an explicit user and inherited test credential"):
|
||||||
|
module.password(args)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
_configure_runtime_environment(monkeypatch)
|
||||||
|
monkeypatch.setenv("HOME", "/tmp/untrusted-home")
|
||||||
|
monkeypatch.setenv("LANG", "attacker-locale")
|
||||||
|
monkeypatch.setenv("SHELL", "/tmp/untrusted-shell")
|
||||||
|
monkeypatch.setenv("UNRELATED_AMBIENT_VALUE", "must-not-reach-psql")
|
||||||
captured: dict[str, object] = {}
|
captured: dict[str, object] = {}
|
||||||
|
|
||||||
def fake_run(command, **kwargs):
|
def fake_run(command, **kwargs):
|
||||||
if command[0] == "gcloud":
|
|
||||||
return SimpleNamespace(returncode=0, stdout="scoped-runtime-password\n", stderr="")
|
|
||||||
captured["command"] = command
|
captured["command"] = command
|
||||||
captured["kwargs"] = kwargs
|
captured["kwargs"] = kwargs
|
||||||
return SimpleNamespace(returncode=0, stdout="ok\n", stderr="")
|
return SimpleNamespace(returncode=0, stdout="ok\n", stderr="")
|
||||||
|
|
||||||
|
monkeypatch.setattr(module, "runtime_password_from_metadata", lambda: "scoped-runtime-password")
|
||||||
monkeypatch.setattr(module.subprocess, "run", fake_run)
|
monkeypatch.setattr(module.subprocess, "run", fake_run)
|
||||||
|
|
||||||
assert module.run_psql(args, "select 1;") == "ok\n"
|
assert module.run_psql(args, "select 1;") == "ok\n"
|
||||||
|
|
@ -324,16 +506,279 @@ def test_cloudsql_psql_uses_only_explicit_pg_environment_fields(monkeypatch: pyt
|
||||||
kwargs = captured["kwargs"]
|
kwargs = captured["kwargs"]
|
||||||
assert isinstance(kwargs, dict)
|
assert isinstance(kwargs, dict)
|
||||||
env = kwargs["env"]
|
env = kwargs["env"]
|
||||||
assert env["PGHOST"] == "10.61.0.3"
|
assert env == {
|
||||||
assert env["PGPORT"] == "5432"
|
"PATH": "/usr/bin:/bin",
|
||||||
assert env["PGDATABASE"] == "teleo_canonical"
|
"PGHOST": "10.61.0.3",
|
||||||
assert env["PGUSER"] == module.RUNTIME_DB_USER
|
"PGPORT": "5432",
|
||||||
assert env["PGSSLMODE"] == "require"
|
"PGDATABASE": "teleo_canonical",
|
||||||
assert env["PGCONNECT_TIMEOUT"] == "8"
|
"PGUSER": module.RUNTIME_DB_USER,
|
||||||
assert env["PGPASSWORD"] == "scoped-runtime-password"
|
"PGSSLMODE": "require",
|
||||||
assert "PGOPTIONS" not in env
|
"PGCONNECT_TIMEOUT": "8",
|
||||||
assert "PGSERVICE" not in env
|
"PGPASSWORD": "scoped-runtime-password",
|
||||||
assert "PGPASSFILE" not in env
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_cloudsql_runtime_metadata_and_secret_requests_are_exact_and_silent(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
capsys: pytest.CaptureFixture[str],
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
_configure_runtime_environment(monkeypatch)
|
||||||
|
bodies = _runtime_http_bodies(module)
|
||||||
|
requests: list[tuple[str, str, dict[str, str], float]] = []
|
||||||
|
|
||||||
|
def fake_urlopen(request, *, timeout):
|
||||||
|
requests.append(
|
||||||
|
(
|
||||||
|
request.full_url,
|
||||||
|
request.get_method(),
|
||||||
|
{key.lower(): value for key, value in request.header_items()},
|
||||||
|
timeout,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return _FakeHttpResponse(bodies.pop(0))
|
||||||
|
|
||||||
|
monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", fake_urlopen)
|
||||||
|
|
||||||
|
assert module.password(args) == "scoped-runtime-password"
|
||||||
|
assert module.GCE_METADATA_EMAIL_URL == (
|
||||||
|
"http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email"
|
||||||
|
)
|
||||||
|
assert module.GCE_METADATA_TOKEN_URL == (
|
||||||
|
"http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token"
|
||||||
|
)
|
||||||
|
assert requests == [
|
||||||
|
(
|
||||||
|
module.GCE_METADATA_EMAIL_URL,
|
||||||
|
"GET",
|
||||||
|
{"metadata-flavor": "Google"},
|
||||||
|
module.HTTP_TIMEOUT_SECONDS,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
module.GCE_METADATA_TOKEN_URL,
|
||||||
|
"GET",
|
||||||
|
{"metadata-flavor": "Google"},
|
||||||
|
module.HTTP_TIMEOUT_SECONDS,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
module.RUNTIME_SECRET_ACCESS_URL,
|
||||||
|
"GET",
|
||||||
|
{"accept": "application/json", "authorization": "Bearer metadata-access-token"},
|
||||||
|
module.HTTP_TIMEOUT_SECONDS,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
assert capsys.readouterr() == ("", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_cloudsql_runtime_http_opener_disables_proxies_and_redirects() -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
proxy_handlers = [
|
||||||
|
handler for handler in module.RUNTIME_HTTP_OPENER.handlers if isinstance(handler, module.urllib.request.ProxyHandler)
|
||||||
|
]
|
||||||
|
redirect_handlers = [
|
||||||
|
handler
|
||||||
|
for handler in module.RUNTIME_HTTP_OPENER.handlers
|
||||||
|
if isinstance(handler, module.urllib.request.HTTPRedirectHandler)
|
||||||
|
]
|
||||||
|
|
||||||
|
# An empty ProxyHandler suppresses build_opener's environment-backed default;
|
||||||
|
# urllib omits the empty handler itself from the final chain.
|
||||||
|
assert proxy_handlers == []
|
||||||
|
assert module.RUNTIME_PROXY_HANDLER.proxies == {}
|
||||||
|
assert len(redirect_handlers) == 1
|
||||||
|
assert type(redirect_handlers[0]) is module._RejectRedirectHandler
|
||||||
|
|
||||||
|
|
||||||
|
def test_cloudsql_runtime_rejects_wrong_metadata_service_account_before_token(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
_configure_runtime_environment(monkeypatch)
|
||||||
|
calls: list[str] = []
|
||||||
|
|
||||||
|
def fake_urlopen(request, *, timeout):
|
||||||
|
assert timeout == module.HTTP_TIMEOUT_SECONDS
|
||||||
|
calls.append(request.full_url)
|
||||||
|
return _FakeHttpResponse(b"unexpected-sa@teleo-501523.iam.gserviceaccount.com")
|
||||||
|
|
||||||
|
monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", fake_urlopen)
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit, match="did not match the required service account"):
|
||||||
|
module.password(args)
|
||||||
|
assert calls == [module.GCE_METADATA_EMAIL_URL]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"token_body",
|
||||||
|
[
|
||||||
|
b"not-json",
|
||||||
|
b"[]",
|
||||||
|
b'{"access_token":"token","token_type":"Basic","expires_in":300}',
|
||||||
|
b'{"access_token":"token with spaces","token_type":"Bearer","expires_in":300}',
|
||||||
|
b'{"access_token":"token","token_type":"Bearer","expires_in":0}',
|
||||||
|
b'{"access_token":"one","access_token":"two","token_type":"Bearer","expires_in":300}',
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_cloudsql_runtime_rejects_malformed_metadata_tokens_without_leaking_them(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
token_body: bytes,
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
_configure_runtime_environment(monkeypatch)
|
||||||
|
bodies = [module.RUNTIME_SERVICE_ACCOUNT.encode("ascii"), token_body]
|
||||||
|
monkeypatch.setattr(
|
||||||
|
module.RUNTIME_HTTP_OPENER,
|
||||||
|
"open",
|
||||||
|
lambda _request, *, timeout: _FakeHttpResponse(bodies.pop(0)),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit, match="Runtime metadata token response was invalid") as captured:
|
||||||
|
module.password(args)
|
||||||
|
assert "token with spaces" not in str(captured.value)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"secret_body",
|
||||||
|
[
|
||||||
|
b"not-json",
|
||||||
|
b"[]",
|
||||||
|
b"{}",
|
||||||
|
b'{"payload":{"data":"%%%"}}',
|
||||||
|
json.dumps({"payload": {"data": base64.b64encode(b"x" * 4097).decode("ascii")}}).encode(),
|
||||||
|
json.dumps({"payload": {"data": base64.b64encode(b"line1\nline2").decode("ascii")}}).encode(),
|
||||||
|
json.dumps({"payload": {"data": base64.b64encode(b"line1\rline2").decode("ascii")}}).encode(),
|
||||||
|
json.dumps({"payload": {"data": base64.b64encode(b"nul\x00byte").decode("ascii")}}).encode(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_cloudsql_runtime_rejects_malformed_secret_payloads_without_leaking_them(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
secret_body: bytes,
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
_configure_runtime_environment(monkeypatch)
|
||||||
|
bodies = _runtime_http_bodies(module)
|
||||||
|
bodies[-1] = secret_body
|
||||||
|
monkeypatch.setattr(
|
||||||
|
module.RUNTIME_HTTP_OPENER,
|
||||||
|
"open",
|
||||||
|
lambda _request, *, timeout: _FakeHttpResponse(bodies.pop(0)),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit, match=r"Scoped runtime secret .* invalid") as captured:
|
||||||
|
module.password(args)
|
||||||
|
assert secret_body.decode("utf-8", errors="ignore") not in str(captured.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cloudsql_runtime_admin_secret_is_impossible_through_arguments(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module, password_secret="gcp-teleo-pgvector-standby-postgres-password")
|
||||||
|
_configure_runtime_environment(monkeypatch)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
module.RUNTIME_HTTP_OPENER,
|
||||||
|
"open",
|
||||||
|
lambda *_args, **_kwargs: pytest.fail("metadata must not be queried for an admin secret argument"),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit, match="requires scoped password secret"):
|
||||||
|
module.password(args)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cloudsql_runtime_http_and_psql_failures_never_echo_credentials(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
_configure_runtime_environment(monkeypatch)
|
||||||
|
dummy = "DUMMY-RUNTIME-SECRET-DO-NOT-LEAK"
|
||||||
|
|
||||||
|
def failed_urlopen(_request, *, timeout):
|
||||||
|
raise RuntimeError(f"provider echoed {dummy}")
|
||||||
|
|
||||||
|
monkeypatch.setattr(module.RUNTIME_HTTP_OPENER, "open", failed_urlopen)
|
||||||
|
with pytest.raises(SystemExit) as http_error:
|
||||||
|
module.password(args)
|
||||||
|
assert dummy not in str(http_error.value)
|
||||||
|
|
||||||
|
monkeypatch.setattr(module, "runtime_password_from_metadata", lambda: dummy)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
module.subprocess,
|
||||||
|
"run",
|
||||||
|
lambda *_args, **_kwargs: SimpleNamespace(returncode=2, stdout=dummy, stderr=dummy),
|
||||||
|
)
|
||||||
|
with pytest.raises(SystemExit, match="Cloud SQL query failed with exit status 2") as psql_error:
|
||||||
|
module.run_psql(args, "select 1;")
|
||||||
|
assert dummy not in str(psql_error.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cloudsql_runtime_read_marker_binds_database_user_private_server_and_ssl(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
marker = {
|
||||||
|
"database": module.RUNTIME_CLOUDSQL_DB,
|
||||||
|
"database_user": module.RUNTIME_DB_USER,
|
||||||
|
"server_addr": module.RUNTIME_CLOUDSQL_HOST,
|
||||||
|
"server_port": int(module.RUNTIME_CLOUDSQL_PORT),
|
||||||
|
"ssl": True,
|
||||||
|
"ssl_version": "TLSv1.3",
|
||||||
|
"system_identifier": "system-id",
|
||||||
|
"wal_lsn": "0/1",
|
||||||
|
}
|
||||||
|
captured: dict[str, object] = {}
|
||||||
|
|
||||||
|
def fake_psql_json_lines(_args, sql, db=None):
|
||||||
|
captured["sql"] = sql
|
||||||
|
captured["db"] = db
|
||||||
|
return [marker]
|
||||||
|
|
||||||
|
monkeypatch.setattr(module, "psql_json_lines", fake_psql_json_lines)
|
||||||
|
|
||||||
|
assert module.database_read_marker(args) == marker
|
||||||
|
assert captured["db"] == module.RUNTIME_CLOUDSQL_DB
|
||||||
|
assert "inet_server_addr()" in str(captured["sql"])
|
||||||
|
assert "pg_stat_ssl" in str(captured["sql"])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("field", "value", "message"),
|
||||||
|
[
|
||||||
|
("database", "teleo_kb", "unexpected database identity"),
|
||||||
|
("database_user", "postgres", "unexpected database identity"),
|
||||||
|
("server_addr", "127.0.0.1", "unexpected server address"),
|
||||||
|
("server_port", 6543, "unexpected server port"),
|
||||||
|
("ssl", False, "encrypted connection"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_cloudsql_runtime_read_marker_rejects_wrong_live_identity(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
field: str,
|
||||||
|
value: object,
|
||||||
|
message: str,
|
||||||
|
) -> None:
|
||||||
|
module = _load_module(BRIDGE_DIR / "cloudsql_memory_tool.py")
|
||||||
|
args = _runtime_connection_args(module)
|
||||||
|
marker = {
|
||||||
|
"database": module.RUNTIME_CLOUDSQL_DB,
|
||||||
|
"database_user": module.RUNTIME_DB_USER,
|
||||||
|
"server_addr": module.RUNTIME_CLOUDSQL_HOST,
|
||||||
|
"server_port": int(module.RUNTIME_CLOUDSQL_PORT),
|
||||||
|
"ssl": True,
|
||||||
|
"ssl_version": "TLSv1.3",
|
||||||
|
"system_identifier": "system-id",
|
||||||
|
"wal_lsn": "0/1",
|
||||||
|
}
|
||||||
|
marker[field] = value
|
||||||
|
monkeypatch.setattr(module, "psql_json_lines", lambda *_args, **_kwargs: [marker])
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit, match=message):
|
||||||
|
module.database_read_marker(args)
|
||||||
|
|
||||||
|
|
||||||
def test_cloudsql_proposal_calls_leave_leo_identity_to_the_database(monkeypatch: pytest.MonkeyPatch) -> None:
|
def test_cloudsql_proposal_calls_leave_leo_identity_to_the_database(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
|
@ -390,6 +835,7 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None:
|
||||||
assert "can_insert_proposals_directly" in sql
|
assert "can_insert_proposals_directly" in sql
|
||||||
assert "can_create_database_objects" in sql
|
assert "can_create_database_objects" in sql
|
||||||
assert "can_create_temp_objects" in sql
|
assert "can_create_temp_objects" in sql
|
||||||
|
assert "can_connect_legacy_database" in sql
|
||||||
assert "unexpectedly has database CREATE" in sql
|
assert "unexpectedly has database CREATE" in sql
|
||||||
assert "REVOKE TEMP FROM PUBLIC would be a" in sql
|
assert "REVOKE TEMP FROM PUBLIC would be a" in sql
|
||||||
assert "can execute unexpected SECURITY DEFINER functions" in sql
|
assert "can execute unexpected SECURITY DEFINER functions" in sql
|
||||||
|
|
@ -407,6 +853,15 @@ def test_gcp_runtime_role_uses_one_narrow_staging_function() -> None:
|
||||||
)
|
)
|
||||||
assert "grant update" not in sql.lower()
|
assert "grant update" not in sql.lower()
|
||||||
assert "grant delete" not in sql.lower()
|
assert "grant delete" not in sql.lower()
|
||||||
|
assert "create role leoclean_kb_runtime nologin" in sql.lower()
|
||||||
|
disable_statement = "alter role leoclean_kb_runtime\n with nologin nosuperuser"
|
||||||
|
password_statement = "alter role leoclean_kb_runtime password"
|
||||||
|
assert sql.lower().index(disable_statement) < sql.lower().index(password_statement)
|
||||||
|
assert sql.lower().index(disable_statement) < sql.index("\\getenv runtime_password")
|
||||||
|
assert sql.index("\\getenv runtime_password") < sql.lower().index(password_statement)
|
||||||
|
assert sql.lower().index("with nologin nosuperuser") < sql.index("\\connect teleo_kb")
|
||||||
|
assert sql.index("\\connect teleo_kb") < sql.lower().rindex("with login nosuperuser")
|
||||||
|
assert "leoclean_kb_runtime final login attributes are unsafe" in sql
|
||||||
|
|
||||||
|
|
||||||
def _load_module(path: Path):
|
def _load_module(path: Path):
|
||||||
|
|
|
||||||
|
|
@ -187,15 +187,60 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped():
|
||||||
assert 'DROPIN_NAME="10-cloudsql-memory.conf"' in script
|
assert 'DROPIN_NAME="10-cloudsql-memory.conf"' in script
|
||||||
assert "20-canonical-kb.conf" not in script
|
assert "20-canonical-kb.conf" not in script
|
||||||
assert "sha256sum" in script
|
assert "sha256sum" in script
|
||||||
|
assert "--preflight-only" in script
|
||||||
assert "--restart" in script
|
assert "--restart" in script
|
||||||
assert 'python3 "$tool_path" status --format json --redacted' in script
|
assert '/usr/bin/python3 "$tool_path" status --format json --redacted' in script
|
||||||
assert "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" in script
|
assert "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" in script
|
||||||
assert "metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" in script
|
assert "169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" in script
|
||||||
assert "env -u PGPASSWORD" in script
|
assert "ProxyHandler({})" in script
|
||||||
assert 'run_scoped_status pre "$remote_tmp/$TOOL_REL"' in script
|
assert "env -i" in script
|
||||||
assert 'run_scoped_status post "$profile_bin/cloudsql_memory_tool.py"' in script
|
assert 'run_scoped_status pre "$payload_dir/$TOOL_REL"' in script
|
||||||
|
assert 'run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL"' in script
|
||||||
|
assert 'sudo -n -u teleo mkdir -p -- "$receipt_dir"' in script
|
||||||
|
assert 'if [ "$PREFLIGHT_ONLY" = true ]; then' in script
|
||||||
|
assert "run_service_wrapper_status post" in script
|
||||||
|
assert "run_service_wrapper_status verify" in script
|
||||||
|
assert '"$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted' in script
|
||||||
|
wrapper_probe = script.split("run_service_wrapper_status() {", maxsplit=1)[1].split(
|
||||||
|
"\n}\n\nservice_fingerprint", maxsplit=1
|
||||||
|
)[0]
|
||||||
|
assert 'nsenter -t "$main_pid" -m --env' in wrapper_probe
|
||||||
|
assert "/usr/bin/setpriv" in wrapper_probe
|
||||||
|
assert "--reuid=teleo" in wrapper_probe
|
||||||
|
assert "TELEO_CLOUDSQL_" not in wrapper_probe
|
||||||
|
assert "/usr/bin/env -i" not in wrapper_probe
|
||||||
|
assert 'run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py"' in script
|
||||||
|
assert 'SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py"' in script
|
||||||
|
assert 'assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py"' in script
|
||||||
|
assert 'assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"' in script
|
||||||
|
assert "REMOTE_CLOUDSDK_CONFIG" not in script
|
||||||
|
assert "--property=DropInPaths" in script
|
||||||
|
assert "--property=EnvironmentFiles" in script
|
||||||
|
assert "Service has an effective EnvironmentFile" in script
|
||||||
|
assert "Service has a later drop-in" in script
|
||||||
|
assert "assert_administrator_secret_denied" in script
|
||||||
|
assert '"stdout_discarded": True' in script
|
||||||
|
assert '"credential_context": "systemd_main_pid_environment"' in script
|
||||||
|
administrator_probe = script.split("assert_administrator_secret_denied() {", maxsplit=1)[1].split(
|
||||||
|
"\n}\n\nrun_scoped_status", maxsplit=1
|
||||||
|
)[0]
|
||||||
|
assert 'nsenter -t "$main_pid" -m --env' in administrator_probe
|
||||||
|
assert "/usr/bin/setpriv" in administrator_probe
|
||||||
|
assert "env -i" not in administrator_probe
|
||||||
|
assert 'preflight_service_before="$(service_fingerprint)"' in script
|
||||||
|
assert 'assert_service_fingerprint "$preflight_service_before"' in script
|
||||||
assert "backup_path" in script
|
assert "backup_path" in script
|
||||||
|
assert "backup_directory_state" in script
|
||||||
assert "restore_path" in script
|
assert "restore_path" in script
|
||||||
|
assert "restore_directory_state" in script
|
||||||
|
assert 'runtime_grandparent="$(dirname "$runtime_parent")"' in script
|
||||||
|
assert 'runtime_anchor="$(dirname "$runtime_grandparent")"' in script
|
||||||
|
assert 'dropin_parent="$(dirname "$dropin_dir")"' in script
|
||||||
|
assert 'backup_directory_state "$runtime_grandparent" runtime_grandparent' in script
|
||||||
|
assert 'restore_directory_state "$runtime_grandparent" runtime_grandparent' in script
|
||||||
|
assert 'sudo test -d "$runtime_anchor"' in script
|
||||||
|
assert 'sudo test -d "$dropin_parent"' in script
|
||||||
|
assert 'sudo rmdir -- "$target"' in script
|
||||||
assert "rollback_runtime" in script
|
assert "rollback_runtime" in script
|
||||||
assert "trap 'on_exit $?' EXIT" in script
|
assert "trap 'on_exit $?' EXIT" in script
|
||||||
assert 'backup_path "$dropin_dir/$DROPIN_NAME" dropin' in script
|
assert 'backup_path "$dropin_dir/$DROPIN_NAME" dropin' in script
|
||||||
|
|
@ -205,13 +250,303 @@ def test_gcp_runtime_sync_is_source_bound_and_scoped():
|
||||||
assert 'sub_state" != "running"' in script
|
assert 'sub_state" != "running"' in script
|
||||||
assert 'sudo systemctl stop "$SERVICE"' in script
|
assert 'sudo systemctl stop "$SERVICE"' in script
|
||||||
assert 'sudo systemctl start "$SERVICE"' in script
|
assert 'sudo systemctl start "$SERVICE"' in script
|
||||||
|
assert 'REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"' in script
|
||||||
|
assert 'sudo install -d -o root -g root -m 0755 "$runtime_parent"' in script
|
||||||
|
assert 'sudo install -d -o root -g root -m 0755 "$runtime_bin"' in script
|
||||||
|
assert 'remote_tmp="$(sudo mktemp -d /var/tmp/teleo-gcp-leoclean-runtime.XXXXXX)"' in script
|
||||||
|
assert 'sudo install -d -o root -g root -m 0755 "$payload_dir"' in script
|
||||||
|
assert 'sudo install -d -o root -g root -m 0700 "$backup_dir"' in script
|
||||||
|
assert 'sudo install -d -o teleo -g teleo -m 0700 "$work_dir"' in script
|
||||||
|
assert "sudo tar --no-same-owner --no-same-permissions" in script
|
||||||
|
assert 'sudo chown -R root:root "$payload_dir"' in script
|
||||||
|
assert 'sudo find "$payload_dir" -type d -exec chmod 0555 {} +' in script
|
||||||
|
assert 'sudo find "$payload_dir" -type l -print -quit' in script
|
||||||
|
assert 'sudo -n -u teleo test ! -w "$payload_dir"' in script
|
||||||
|
assert 'sudo -n -u teleo test ! -w "$backup_dir"' in script
|
||||||
|
assert 'sudo touch "$backup_dir/$name.present"' in script
|
||||||
|
assert 'sudo test -f "$backup_dir/$name.present"' in script
|
||||||
|
assert 'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage"' in script
|
||||||
|
assert 'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage"' in script
|
||||||
|
assert (
|
||||||
|
'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" '
|
||||||
|
'"$permission_verifier_stage"' in script
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" '
|
||||||
|
'"$service_env_verifier_stage"' in script
|
||||||
|
)
|
||||||
|
assert 'assert_installed_files\nsudo systemctl daemon-reload' in script
|
||||||
|
assert 'tar -C "$remote_tmp" -xf -' not in script
|
||||||
|
assert 'mkdir -p "$backup_dir"' not in script
|
||||||
|
assert 'for protected_file in \\\n "$wrapper" \\' in script
|
||||||
|
assert '"$permission_verifier" \\' in script
|
||||||
|
assert '"$service_env_verifier" \\' in script
|
||||||
|
assert 'sudo -n -u teleo test ! -w "$protected_file"' in script
|
||||||
|
assert 'for protected_dir in \\\n /usr \\' in script
|
||||||
|
assert "/usr/local/libexec/livingip" in script
|
||||||
|
assert 'sudo -n -u teleo test ! -w "$protected_dir"' in script
|
||||||
|
assert 'for executable_dir in \\\n /usr/local/sbin \\' in script
|
||||||
|
assert 'sudo stat -Lc \'%U:%G\' "$executable_dir"' in script
|
||||||
|
assert 'sudo -n -u teleo test ! -w "$executable_dir"' in script
|
||||||
|
assert 'for protected_file in /bin/bash /usr/bin/curl /usr/bin/python3' in script
|
||||||
|
assert '/usr/bin/curl -q --noproxy' in script
|
||||||
|
assert 'test "$(sudo cat "$revision")" = "$SOURCE_REVISION"' in script
|
||||||
|
assert 'assert_existing_root_directory "$existing_directory"' in script
|
||||||
|
assert 'assert_existing_root_file "$existing_file"' in script
|
||||||
|
assert 'validate_restored_path "$runtime_bin/teleo-kb" wrapper' in script
|
||||||
|
assert 'validate_restored_directory "$runtime_bin" runtime_bin' in script
|
||||||
|
rollback = script.split("rollback_runtime() {", maxsplit=1)[1].split("\n}\n\non_exit()", maxsplit=1)[0]
|
||||||
|
assert rollback.index("validate_restored_path") < rollback.index("systemctl daemon-reload")
|
||||||
|
assert rollback.index("systemctl daemon-reload") < rollback.index('systemctl start "$SERVICE"')
|
||||||
|
assert rollback.count('if [ "$rollback_rc" -eq 0 ]; then') >= 3
|
||||||
|
deploy_tail = script.split("mutation_started=true", maxsplit=1)[1]
|
||||||
|
assert deploy_tail.index("assert_installed_files") < deploy_tail.index("systemctl daemon-reload")
|
||||||
|
assert deploy_tail.index("systemctl daemon-reload") < deploy_tail.index("assert_unit_configuration")
|
||||||
|
assert deploy_tail.index("assert_unit_configuration") < deploy_tail.index('systemctl start "$SERVICE"')
|
||||||
|
for install_line in (
|
||||||
|
'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL"',
|
||||||
|
'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL"',
|
||||||
|
'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL"',
|
||||||
|
'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL"',
|
||||||
|
'sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL"',
|
||||||
|
):
|
||||||
|
assert install_line in deploy_tail
|
||||||
|
assert 'sudo install -o root -g root -m 0755 "$work_dir/' not in script
|
||||||
|
assert 'nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb"' in script
|
||||||
|
assert 'findmnt -T "$SERVICE_PROFILE_BIN"' in script
|
||||||
|
assert 'findmnt -M "$SERVICE_HERMES_ROOT"' in script
|
||||||
|
assert 'findmnt -M "$SERVICE_PROFILE_ROOT/state"' in script
|
||||||
|
assert 'findmnt -M "$SERVICE_PROFILE_ROOT/workspace"' in script
|
||||||
|
assert "cap_sys_admin" in script
|
||||||
|
assert '"current_tier": "T3_live_readonly"' in script
|
||||||
|
assert '"status": "pass"' in script
|
||||||
assert "UnsetEnvironment=PGPASSWORD" in dropin
|
assert "UnsetEnvironment=PGPASSWORD" in dropin
|
||||||
assert "RuntimeDirectory=leoclean-gcp-prod-parallel" in dropin
|
assert "UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CONFIG" in dropin
|
||||||
assert "RuntimeDirectoryMode=0700" in dropin
|
assert "UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS" in dropin
|
||||||
assert "Environment=CLOUDSDK_CONFIG=/run/leoclean-gcp-prod-parallel" in dropin
|
assert "UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT" in dropin
|
||||||
|
assert "UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT" in dropin
|
||||||
|
assert "UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY" in dropin
|
||||||
|
assert "UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE" in dropin
|
||||||
|
assert (
|
||||||
|
"BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin" in dropin
|
||||||
|
)
|
||||||
|
assert "ReadOnlyPaths=/home/teleo/.hermes" in dropin
|
||||||
|
assert (
|
||||||
|
"ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state "
|
||||||
|
"/home/teleo/.hermes/profiles/leoclean/workspace" in dropin
|
||||||
|
)
|
||||||
|
assert "CapabilityBoundingSet=~CAP_SYS_ADMIN" in dropin
|
||||||
|
assert "RuntimeDirectory=" not in dropin
|
||||||
|
assert "Environment=CLOUDSDK_" not in dropin
|
||||||
|
assert "Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" in dropin
|
||||||
assert "TELEO_KB_MODE=cloudsql" in dropin
|
assert "TELEO_KB_MODE=cloudsql" in dropin
|
||||||
|
assert "TELEO_GCP_METADATA_ONLY=1" in dropin
|
||||||
assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in dropin
|
assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in dropin
|
||||||
assert "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" in dropin
|
assert "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" in dropin
|
||||||
assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in dropin
|
assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in dropin
|
||||||
assert not any(line.startswith("Environment=PGPASSWORD") for line in dropin.splitlines())
|
assert not any(line.startswith("Environment=PGPASSWORD") for line in dropin.splitlines())
|
||||||
assert "gcp-teleo-pgvector-standby-postgres-password" not in dropin
|
assert "gcp-teleo-pgvector-standby-postgres-password" not in dropin
|
||||||
|
|
||||||
|
|
||||||
|
def test_gcp_runtime_embedded_remote_programs_are_bash_syntax_valid() -> None:
|
||||||
|
script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text()
|
||||||
|
blocks: dict[str, str] = {}
|
||||||
|
|
||||||
|
for variable in ("remote_helpers", "verify_body", "sync_body"):
|
||||||
|
marker = f"{variable}=$(cat <<'REMOTE'\n"
|
||||||
|
assert script.count(marker) == 1
|
||||||
|
start = script.index(marker) + len(marker)
|
||||||
|
end = script.index("\nREMOTE\n)", start)
|
||||||
|
blocks[variable] = script[start:end]
|
||||||
|
|
||||||
|
for name, body in (
|
||||||
|
*blocks.items(),
|
||||||
|
("composed_verify", blocks["remote_helpers"] + "\n" + blocks["verify_body"]),
|
||||||
|
("composed_sync", blocks["remote_helpers"] + "\n" + blocks["sync_body"]),
|
||||||
|
):
|
||||||
|
completed = subprocess.run(
|
||||||
|
["bash", "-n"],
|
||||||
|
input=body,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert completed.returncode == 0, f"{name}: {completed.stderr}"
|
||||||
|
|
||||||
|
embedded_python = [part.split("\nPY", maxsplit=1)[0] for part in blocks["remote_helpers"].split("<<'PY'\n")[1:]]
|
||||||
|
assert len(embedded_python) == 2
|
||||||
|
for index, body in enumerate(embedded_python):
|
||||||
|
completed = subprocess.run(
|
||||||
|
["python3", "-c", "import sys; compile(sys.stdin.read(), '<embedded>', 'exec')"],
|
||||||
|
input=body,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert completed.returncode == 0, f"embedded_python_{index}: {completed.stderr}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_gcp_runtime_rejects_preexisting_symlink_before_mutation(tmp_path: Path) -> None:
|
||||||
|
script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text()
|
||||||
|
function_start = script.index("assert_existing_root_directory() {")
|
||||||
|
function_end = script.index("\n\nbackup_path() {", function_start)
|
||||||
|
trust_functions = script[function_start:function_end]
|
||||||
|
real_directory = tmp_path / "real-runtime"
|
||||||
|
real_directory.mkdir()
|
||||||
|
symlink = tmp_path / "runtime-link"
|
||||||
|
symlink.symlink_to(real_directory, target_is_directory=True)
|
||||||
|
harness = f"""
|
||||||
|
set -euo pipefail
|
||||||
|
sudo() {{
|
||||||
|
if [ "${{1:-}}" = -n ]; then
|
||||||
|
shift
|
||||||
|
if [ "${{1:-}}" = -u ]; then shift 2; fi
|
||||||
|
fi
|
||||||
|
"$@"
|
||||||
|
}}
|
||||||
|
{trust_functions}
|
||||||
|
assert_existing_root_directory "$1"
|
||||||
|
"""
|
||||||
|
completed = subprocess.run(
|
||||||
|
["bash", "-s", "--", str(symlink)],
|
||||||
|
input=harness,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert completed.returncode != 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_gcp_runtime_executable_configuration_override_gate() -> None:
|
||||||
|
script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text()
|
||||||
|
function_start = script.index("assert_unit_configuration() {")
|
||||||
|
function_end = script.index("\n\nassert_runtime_environment() {", function_start)
|
||||||
|
unit_configuration_function = script[function_start:function_end]
|
||||||
|
harness = f"""
|
||||||
|
set -euo pipefail
|
||||||
|
SERVICE=leoclean.service
|
||||||
|
REMOTE_DROPIN_DIR=/etc/systemd/system/leoclean.service.d
|
||||||
|
DROPIN_NAME=10-cloudsql-memory.conf
|
||||||
|
systemctl() {{
|
||||||
|
case "$*" in
|
||||||
|
*--property=DropInPaths*) printf '%s\\n' "${{FAKE_DROPINS:-}}" ;;
|
||||||
|
*--property=EnvironmentFiles*) printf '%s\\n' "${{FAKE_ENVIRONMENT_FILES:-}}" ;;
|
||||||
|
*) return 90 ;;
|
||||||
|
esac
|
||||||
|
}}
|
||||||
|
{unit_configuration_function}
|
||||||
|
assert_unit_configuration
|
||||||
|
"""
|
||||||
|
reviewed = "/etc/systemd/system/leoclean.service.d/10-cloudsql-memory.conf"
|
||||||
|
cases = (
|
||||||
|
{
|
||||||
|
"FAKE_DROPINS": f"{reviewed} /etc/systemd/system/leoclean.service.d/99-conflict.conf",
|
||||||
|
"FAKE_ENVIRONMENT_FILES": "",
|
||||||
|
},
|
||||||
|
{"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": "/etc/leoclean-runtime.env"},
|
||||||
|
)
|
||||||
|
for environment in cases:
|
||||||
|
completed = subprocess.run(
|
||||||
|
["bash", "-s"],
|
||||||
|
input=harness,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
env=environment,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert completed.returncode != 0
|
||||||
|
|
||||||
|
allowed = subprocess.run(
|
||||||
|
["bash", "-s"],
|
||||||
|
input=harness,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
env={"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": ""},
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert allowed.returncode == 0, allowed.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_gcp_runtime_absence_rollback_removes_every_created_directory(tmp_path: Path) -> None:
|
||||||
|
script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text()
|
||||||
|
function_start = script.index("backup_directory_state() {")
|
||||||
|
function_end = script.index("\nrestore_path() {", function_start)
|
||||||
|
directory_functions = script[function_start:function_end]
|
||||||
|
harness = f"""
|
||||||
|
set -euo pipefail
|
||||||
|
sudo() {{ "$@"; }}
|
||||||
|
root="$1"
|
||||||
|
backup_dir="$root/backup"
|
||||||
|
mkdir -p "$backup_dir" "$root/usr/local" "$root/etc/systemd/system"
|
||||||
|
runtime_grandparent="$root/usr/local/libexec"
|
||||||
|
runtime_parent="$runtime_grandparent/livingip"
|
||||||
|
runtime_bin="$runtime_parent/leoclean-kb"
|
||||||
|
dropin_dir="$root/etc/systemd/system/service.d"
|
||||||
|
{directory_functions}
|
||||||
|
backup_directory_state "$runtime_grandparent" runtime_grandparent
|
||||||
|
backup_directory_state "$runtime_parent" runtime_parent
|
||||||
|
backup_directory_state "$runtime_bin" runtime_bin
|
||||||
|
backup_directory_state "$dropin_dir" dropin_dir
|
||||||
|
mkdir -p "$runtime_bin" "$dropin_dir"
|
||||||
|
restore_directory_state "$runtime_bin" runtime_bin
|
||||||
|
restore_directory_state "$runtime_parent" runtime_parent
|
||||||
|
restore_directory_state "$runtime_grandparent" runtime_grandparent
|
||||||
|
restore_directory_state "$dropin_dir" dropin_dir
|
||||||
|
test ! -e "$runtime_grandparent"
|
||||||
|
test ! -e "$dropin_dir"
|
||||||
|
test -d "$root/usr/local"
|
||||||
|
test -d "$root/etc/systemd/system"
|
||||||
|
"""
|
||||||
|
completed = subprocess.run(
|
||||||
|
["bash", "-s", "--", str(tmp_path)],
|
||||||
|
input=harness,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert completed.returncode == 0, completed.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_gcp_runtime_rollback_never_reloads_or_restarts_after_validation_failure(tmp_path: Path) -> None:
|
||||||
|
script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text()
|
||||||
|
function_start = script.index("rollback_runtime() {")
|
||||||
|
function_end = script.index("\n\non_exit() {", function_start)
|
||||||
|
rollback_function = script[function_start:function_end]
|
||||||
|
action_log = tmp_path / "systemctl-actions"
|
||||||
|
harness = f"""
|
||||||
|
set -u
|
||||||
|
SERVICE=leoclean.service
|
||||||
|
runtime_bin=/usr/local/libexec/livingip/leoclean-kb
|
||||||
|
runtime_parent=/usr/local/libexec/livingip
|
||||||
|
runtime_grandparent=/usr/local/libexec
|
||||||
|
dropin_dir=/etc/systemd/system/leoclean.service.d
|
||||||
|
DROPIN_NAME=10-cloudsql-memory.conf
|
||||||
|
action_log="$1"
|
||||||
|
sudo() {{
|
||||||
|
if [ "${{1:-}}" = systemctl ]; then
|
||||||
|
printf '%s\\n' "$2" >>"$action_log"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
"$@"
|
||||||
|
}}
|
||||||
|
restore_path() {{ return 0; }}
|
||||||
|
restore_directory_state() {{ return 0; }}
|
||||||
|
validate_restored_path() {{ [ "$2" != wrapper ]; }}
|
||||||
|
validate_restored_directory() {{ return 0; }}
|
||||||
|
assert_unit_configuration() {{ printf '%s\\n' unit-configuration >>"$action_log"; }}
|
||||||
|
assert_service_running() {{ printf '%s\\n' assert-running >>"$action_log"; }}
|
||||||
|
{rollback_function}
|
||||||
|
rollback_runtime
|
||||||
|
"""
|
||||||
|
completed = subprocess.run(
|
||||||
|
["bash", "-s", "--", str(action_log)],
|
||||||
|
input=harness,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert completed.returncode != 0
|
||||||
|
assert action_log.read_text(encoding="utf-8").splitlines() == ["stop"]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue