teleo-infrastructure/ops/check_observatory_read_adapter_gcp_preflight.py
twentyOne2x 36fe0d3cc2
Some checks are pending
CI / lint-and-test (push) Waiting to run
Harden Observatory private staging deploy gate (#163)
Merge the T2 readiness and least-privilege staging gate. This does not deploy Cloud Run, mutate Cloud SQL, or change production routing.
2026-07-15 09:55:49 +02:00

669 lines
23 KiB
Python

#!/usr/bin/env python3
"""Run the adapter-specific, non-mutating GCP staging preflight."""
from __future__ import annotations
import argparse
import json
import re
import subprocess
from collections.abc import Callable
from dataclasses import asdict, dataclass
from pathlib import Path
PROJECT = "teleo-501523"
PROJECT_NUMBER = "785938879453"
REGION = "europe-west6"
NETWORK = "teleo-staging-net"
SUBNET = "teleo-staging-europe-west6"
INSTANCE = "teleo-pgvector-standby"
SERVICE = "observatory-read-adapter-staging"
DEPLOY_SERVICE_ACCOUNT = f"sa-observatory-deployer@{PROJECT}.iam.gserviceaccount.com"
RUNTIME_SERVICE_ACCOUNT = f"sa-observatory-read-adapter@{PROJECT}.iam.gserviceaccount.com"
BUILD_SERVICE_ACCOUNT = f"sa-artifact-builder@{PROJECT}.iam.gserviceaccount.com"
DB_IAM_USER = f"sa-observatory-read-adapter@{PROJECT}.iam"
SECRET = "observatory-read-api-key-staging"
REPOSITORY = "teleo"
WIF_POOL = "github-actions"
DEPLOY_WIF_PROVIDER = "observatory-read-adapter-main"
DEPLOY_WORKFLOW_REF = (
"living-ip/teleo-infrastructure/.github/workflows/"
"gcp-observatory-read-adapter.yml@refs/heads/main"
)
DEPLOY_WIF_ATTRIBUTE_CONDITION = (
"assertion.repository=='living-ip/teleo-infrastructure' && "
"assertion.ref=='refs/heads/main' && "
f"assertion.workflow_ref=='{DEPLOY_WORKFLOW_REF}'"
)
DEPLOY_WIF_MEMBER = (
f"principal://iam.googleapis.com/projects/{PROJECT_NUMBER}/locations/global/"
f"workloadIdentityPools/{WIF_POOL}/subject/"
"repo:living-ip/teleo-infrastructure:ref:refs/heads/main"
)
PREFLIGHT_ROLE = f"projects/{PROJECT}/roles/observatoryStagingPreflight"
DEPLOY_ROLE = f"projects/{PROJECT}/roles/observatoryStagingDeployer"
PREFLIGHT_PERMISSIONS = {
"artifactregistry.repositories.getIamPolicy",
"cloudsql.instances.get",
"cloudsql.users.list",
"compute.networks.get",
"compute.subnetworks.get",
"iam.serviceAccounts.get",
"iam.serviceAccounts.getIamPolicy",
"iam.roles.get",
"iam.workloadIdentityPoolProviders.get",
"resourcemanager.projects.get",
"resourcemanager.projects.getIamPolicy",
"run.operations.get",
"run.services.get",
"run.services.getIamPolicy",
"secretmanager.secrets.getIamPolicy",
"serviceusage.services.get",
"serviceusage.services.use",
}
DEPLOY_PERMISSIONS = {
"run.operations.get",
"run.services.create",
"run.services.get",
"run.services.setIamPolicy",
"run.services.update",
}
REQUIRED_APIS = (
"artifactregistry.googleapis.com",
"compute.googleapis.com",
"iamcredentials.googleapis.com",
"run.googleapis.com",
"secretmanager.googleapis.com",
"sqladmin.googleapis.com",
"sts.googleapis.com",
)
IMAGE_RE = re.compile(
rf"^europe-west6-docker[.]pkg[.]dev/{PROJECT}/teleo/observatory-read-adapter:"
r"[0-9a-f]{40}-[0-9]+-[1-9][0-9]*@sha256:[0-9a-f]{64}$"
)
@dataclass(frozen=True)
class Check:
name: str
status: str
detail: str
def sanitize_error(value: str) -> str:
value = re.sub(r"/[^\s]*/gha-creds-[^\s]+[.]json", "<ephemeral-wif-credential>", value)
return " ".join(value.strip().split())[-600:] or "command_failed_without_stderr"
def run(command: list[str]) -> subprocess.CompletedProcess[str]:
try:
return subprocess.run(command, text=True, capture_output=True, timeout=30, check=False)
except subprocess.TimeoutExpired:
return subprocess.CompletedProcess(command, 124, stdout="", stderr="command_timed_out_after_30_seconds")
except OSError as exc:
return subprocess.CompletedProcess(command, 127, stdout="", stderr=f"command_unavailable:{exc}")
def command_check(
name: str,
command: list[str],
validate: Callable[[str], bool],
success_detail: str,
) -> Check:
result = run(command)
if result.returncode != 0:
return Check(name, "blocked", sanitize_error(result.stderr))
try:
valid = validate(result.stdout)
except (AttributeError, IndexError, KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
return Check(name, "blocked", f"invalid_readback:{type(exc).__name__}")
return Check(name, "pass" if valid else "blocked", success_detail if valid else "unexpected_readback")
def cloud_sql_is_private_and_iam_enabled(value: str) -> bool:
payload = json.loads(value)
flags = {
item.get("name"): str(item.get("value", "")).lower()
for item in payload.get("settings", {}).get("databaseFlags", [])
}
ip_configuration = payload.get("settings", {}).get("ipConfiguration", {})
address_types = {item.get("type") for item in payload.get("ipAddresses", [])}
return (
payload.get("name") == INSTANCE
and payload.get("region") == REGION
and str(payload.get("databaseVersion", "")).startswith("POSTGRES_")
and ip_configuration.get("ipv4Enabled") is False
and str(ip_configuration.get("privateNetwork", "")).endswith(f"/networks/{NETWORK}")
and "PRIVATE" in address_types
and "PRIMARY" not in address_types
and flags.get("cloudsql.iam_authentication") == "on"
)
def cloud_sql_iam_user_exists(value: str) -> bool:
users = json.loads(value)
return any(
user.get("name") == DB_IAM_USER and user.get("type") == "CLOUD_IAM_SERVICE_ACCOUNT" for user in users
)
def subnet_is_exact(value: str) -> bool:
payload = json.loads(value)
return (
payload.get("name") == SUBNET
and str(payload.get("region", "")).endswith(f"/regions/{REGION}")
and str(payload.get("network", "")).endswith(f"/networks/{NETWORK}")
)
def secret_is_valid(value: str) -> bool:
try:
encoded = value.encode("ascii")
except UnicodeEncodeError:
return False
return value == value.strip() and 32 <= len(encoded) <= 512
def normalized(value: str) -> str:
return " ".join(value.split())
def policy_has_binding(
policy: dict[str, object],
role: str,
member: str,
*,
condition_title: str | None = None,
condition_expression: str | None = None,
) -> bool:
bindings = policy.get("bindings", [])
if not isinstance(bindings, list):
return False
for binding in bindings:
if not isinstance(binding, dict) or binding.get("role") != role:
continue
members = binding.get("members", [])
if not isinstance(members, list) or member not in members:
continue
condition = binding.get("condition")
if condition_title is None and condition_expression is None:
return condition in (None, {})
if not isinstance(condition, dict):
continue
if condition_title is not None and condition.get("title") != condition_title:
continue
if condition_expression is not None and normalized(str(condition.get("expression", ""))) != normalized(
condition_expression
):
continue
return True
return False
def member_roles(policy: dict[str, object], member: str) -> set[str]:
roles: set[str] = set()
bindings = policy.get("bindings", [])
if not isinstance(bindings, list):
return roles
for binding in bindings:
if not isinstance(binding, dict):
continue
members = binding.get("members", [])
role = binding.get("role")
if isinstance(members, list) and member in members and isinstance(role, str):
roles.add(role)
return roles
def project_iam_is_exact(value: str) -> bool:
policy = json.loads(value)
deployer = f"serviceAccount:{DEPLOY_SERVICE_ACCOUNT}"
runtime = f"serviceAccount:{RUNTIME_SERVICE_ACCOUNT}"
builder = f"serviceAccount:{BUILD_SERVICE_ACCOUNT}"
service_agent = (
f"serviceAccount:service-{PROJECT_NUMBER}@serverless-robot-prod.iam.gserviceaccount.com"
)
sql_expression = (
f"resource.name == 'projects/{PROJECT}/instances/{INSTANCE}' && "
"resource.service == 'sqladmin.googleapis.com'"
)
forbidden_deployer = {
"roles/artifactregistry.writer",
"roles/cloudsql.admin",
"roles/compute.admin",
"roles/editor",
"roles/owner",
"roles/run.admin",
"roles/run.developer",
"roles/secretmanager.admin",
}
forbidden_builder = forbidden_deployer | {
DEPLOY_ROLE,
PREFLIGHT_ROLE,
"roles/iam.serviceAccountUser",
"roles/secretmanager.secretAccessor",
}
return (
policy_has_binding(policy, PREFLIGHT_ROLE, deployer)
and policy_has_binding(policy, DEPLOY_ROLE, deployer)
and policy_has_binding(
policy,
"roles/cloudsql.client",
runtime,
condition_title="observatory-exact-cloudsql-instance",
condition_expression=sql_expression,
)
and policy_has_binding(
policy,
"roles/cloudsql.instanceUser",
runtime,
condition_title="observatory-exact-cloudsql-instance",
condition_expression=sql_expression,
)
and policy_has_binding(policy, "roles/run.serviceAgent", service_agent)
and not (member_roles(policy, deployer) & forbidden_deployer)
and not (member_roles(policy, builder) & forbidden_builder)
)
def role_has_exact_permissions(value: str, expected: set[str]) -> bool:
payload = json.loads(value)
permissions = payload.get("includedPermissions", [])
return isinstance(permissions, list) and set(permissions) == expected and payload.get("deleted") is not True
def wif_provider_is_exact(value: str) -> bool:
payload = json.loads(value)
mapping = payload.get("attributeMapping", {})
expected_mapping = {
"google.subject": "assertion.sub",
"attribute.repository": "assertion.repository",
"attribute.ref": "assertion.ref",
"attribute.workflow_ref": "assertion.workflow_ref",
}
return (
payload.get("state") == "ACTIVE"
and mapping == expected_mapping
and normalized(str(payload.get("attributeCondition", ""))) == normalized(DEPLOY_WIF_ATTRIBUTE_CONDITION)
)
def deployer_service_account_policy_is_exact(value: str) -> bool:
policy = json.loads(value)
return policy_has_binding(policy, "roles/iam.workloadIdentityUser", DEPLOY_WIF_MEMBER)
def runtime_service_account_policy_is_exact(value: str) -> bool:
policy = json.loads(value)
deployer = f"serviceAccount:{DEPLOY_SERVICE_ACCOUNT}"
builder = f"serviceAccount:{BUILD_SERVICE_ACCOUNT}"
return policy_has_binding(policy, "roles/iam.serviceAccountUser", deployer) and not member_roles(policy, builder)
def artifact_policy_is_exact(value: str) -> bool:
policy = json.loads(value)
deployer = f"serviceAccount:{DEPLOY_SERVICE_ACCOUNT}"
return policy_has_binding(policy, "roles/artifactregistry.reader", deployer)
def secret_policy_is_exact(value: str) -> bool:
policy = json.loads(value)
deployer = f"serviceAccount:{DEPLOY_SERVICE_ACCOUNT}"
runtime = f"serviceAccount:{RUNTIME_SERVICE_ACCOUNT}"
builder = f"serviceAccount:{BUILD_SERVICE_ACCOUNT}"
return (
policy_has_binding(policy, "roles/secretmanager.secretAccessor", deployer)
and policy_has_binding(policy, "roles/secretmanager.secretAccessor", runtime)
and policy_has_binding(policy, "roles/secretmanager.viewer", deployer)
and not member_roles(policy, builder)
)
def api_key_secret_check() -> Check:
version_result = run(
[
"gcloud",
"secrets",
"versions",
"list",
SECRET,
"--project",
PROJECT,
"--filter=state=ENABLED",
"--sort-by=~createTime",
"--limit=1",
"--format=value(name)",
]
)
if version_result.returncode != 0:
return Check("api_key_secret", "blocked", sanitize_error(version_result.stderr))
version = version_result.stdout.strip().rsplit("/", 1)[-1]
if not version.isdigit():
return Check("api_key_secret", "blocked", "enabled_secret_version_missing")
secret_result = run(
[
"gcloud",
"secrets",
"versions",
"access",
version,
"--secret",
SECRET,
"--project",
PROJECT,
]
)
if secret_result.returncode != 0:
return Check("api_key_secret", "blocked", sanitize_error(secret_result.stderr))
valid = secret_is_valid(secret_result.stdout)
detail = f"enabled_version_{version}_valid_length_value_redacted" if valid else "invalid_secret_value"
return Check("api_key_secret", "pass" if valid else "blocked", detail)
def existing_service_policy_check() -> Check:
describe = run(
[
"gcloud",
"run",
"services",
"describe",
SERVICE,
"--project",
PROJECT,
"--region",
REGION,
"--format=value(metadata.name)",
]
)
if describe.returncode != 0:
error = sanitize_error(describe.stderr)
if "not found" in error.lower() or "not_found" in error.lower():
return Check("existing_service_policy", "pass", "service_not_created_yet")
return Check("existing_service_policy", "blocked", error)
if describe.stdout.strip() != SERVICE:
return Check("existing_service_policy", "blocked", "unexpected_service_readback")
policy_result = run(
[
"gcloud",
"run",
"services",
"get-iam-policy",
SERVICE,
"--project",
PROJECT,
"--region",
REGION,
"--format=json",
]
)
if policy_result.returncode != 0:
return Check("existing_service_policy", "blocked", sanitize_error(policy_result.stderr))
try:
policy = json.loads(policy_result.stdout)
builder = f"serviceAccount:{BUILD_SERVICE_ACCOUNT}"
valid = not member_roles(policy, builder)
except (AttributeError, TypeError, ValueError, json.JSONDecodeError) as exc:
return Check("existing_service_policy", "blocked", f"invalid_readback:{type(exc).__name__}")
return Check(
"existing_service_policy",
"pass" if valid else "blocked",
"artifact_builder_absent" if valid else "artifact_builder_has_service_binding",
)
def build_checks(image_ref: str) -> list[Check]:
image_name, image_digest = image_ref.rsplit("@", 1) if "@" in image_ref else (image_ref, "")
digest_ref = f"{image_name.rsplit(':', 1)[0]}@{image_digest}" if image_digest else image_ref
checks = [
command_check(
"active_identity",
["gcloud", "auth", "list", "--filter=status:ACTIVE", "--format=value(account)"],
lambda value: value.strip() == DEPLOY_SERVICE_ACCOUNT,
"dedicated_observatory_deployer",
),
command_check(
"project",
["gcloud", "projects", "describe", PROJECT, "--format=value(projectId)"],
lambda value: value.strip() == PROJECT,
PROJECT,
),
]
checks.extend(
command_check(
f"api_{api.removesuffix('.googleapis.com')}",
["gcloud", "services", "describe", api, "--project", PROJECT, "--format=value(state)"],
lambda value: value.strip() == "ENABLED",
"enabled",
)
for api in REQUIRED_APIS
)
checks.extend(
[
command_check(
"deployer_wif_provider",
[
"gcloud",
"iam",
"workload-identity-pools",
"providers",
"describe",
DEPLOY_WIF_PROVIDER,
"--project",
PROJECT,
"--location",
"global",
"--workload-identity-pool",
WIF_POOL,
"--format=json",
],
wif_provider_is_exact,
"main_and_exact_workflow_only",
),
command_check(
"preflight_custom_role",
[
"gcloud",
"iam",
"roles",
"describe",
"observatoryStagingPreflight",
"--project",
PROJECT,
"--format=json",
],
lambda value: role_has_exact_permissions(value, PREFLIGHT_PERMISSIONS),
"exact_read_only_permissions",
),
command_check(
"deploy_custom_role",
[
"gcloud",
"iam",
"roles",
"describe",
"observatoryStagingDeployer",
"--project",
PROJECT,
"--format=json",
],
lambda value: role_has_exact_permissions(value, DEPLOY_PERMISSIONS),
"five_permissions_no_delete",
),
command_check(
"project_iam",
["gcloud", "projects", "get-iam-policy", PROJECT, "--format=json"],
project_iam_is_exact,
"required_bindings_present_broad_roles_absent",
),
command_check(
"deployer_wif_binding",
[
"gcloud",
"iam",
"service-accounts",
"get-iam-policy",
DEPLOY_SERVICE_ACCOUNT,
"--project",
PROJECT,
"--format=json",
],
deployer_service_account_policy_is_exact,
"exact_main_subject_workload_identity_user",
),
command_check(
"runtime_act_as_policy",
[
"gcloud",
"iam",
"service-accounts",
"get-iam-policy",
RUNTIME_SERVICE_ACCOUNT,
"--project",
PROJECT,
"--format=json",
],
runtime_service_account_policy_is_exact,
"deployer_only_artifact_builder_absent",
),
command_check(
"artifact_repository_policy",
[
"gcloud",
"artifacts",
"repositories",
"get-iam-policy",
REPOSITORY,
"--project",
PROJECT,
"--location",
REGION,
"--format=json",
],
artifact_policy_is_exact,
"deployer_reader",
),
command_check(
"secret_policy",
[
"gcloud",
"secrets",
"get-iam-policy",
SECRET,
"--project",
PROJECT,
"--format=json",
],
secret_policy_is_exact,
"runtime_and_deployer_only_artifact_builder_absent",
),
existing_service_policy_check(),
]
)
checks.extend(
[
command_check(
"runtime_service_account",
[
"gcloud",
"iam",
"service-accounts",
"describe",
RUNTIME_SERVICE_ACCOUNT,
"--project",
PROJECT,
"--format=value(email)",
],
lambda value: value.strip() == RUNTIME_SERVICE_ACCOUNT,
RUNTIME_SERVICE_ACCOUNT,
),
command_check(
"vpc_network",
[
"gcloud",
"compute",
"networks",
"describe",
NETWORK,
"--project",
PROJECT,
"--format=value(name)",
],
lambda value: value.strip() == NETWORK,
NETWORK,
),
command_check(
"vpc_subnet",
[
"gcloud",
"compute",
"networks",
"subnets",
"describe",
SUBNET,
"--project",
PROJECT,
"--region",
REGION,
"--format=json",
],
subnet_is_exact,
f"{NETWORK}/{REGION}/{SUBNET}",
),
command_check(
"cloud_sql_private_iam",
["gcloud", "sql", "instances", "describe", INSTANCE, "--project", PROJECT, "--format=json"],
cloud_sql_is_private_and_iam_enabled,
f"{PROJECT}:{REGION}:{INSTANCE}:private_ip_only:iam_authentication_on",
),
command_check(
"cloud_sql_iam_user",
["gcloud", "sql", "users", "list", "--instance", INSTANCE, "--project", PROJECT, "--format=json"],
cloud_sql_iam_user_exists,
DB_IAM_USER,
),
api_key_secret_check(),
command_check(
"immutable_image",
["gcloud", "artifacts", "docker", "images", "describe", digest_ref, "--project", PROJECT, "--format=json"],
lambda _value: bool(IMAGE_RE.fullmatch(image_ref)),
image_digest or "missing_digest",
),
]
)
return checks
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--image-ref", required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
checks = build_checks(args.image_ref)
pass_count = sum(check.status == "pass" for check in checks)
blocked_count = len(checks) - pass_count
receipt = {
"schema": "livingip.observatory-read-adapter-gcp-preflight.v1",
"project": PROJECT,
"required_tier": "T3_live_readonly",
"current_tier": "T2_authenticated_gcp_preflight",
"ready_for_staging_deploy": blocked_count == 0,
"claim_ceiling": "Preflight only; T3 requires the live private Cloud SQL claim and negative receipts.",
"identity": DEPLOY_SERVICE_ACCOUNT,
"checks": [asdict(check) for check in checks],
"pass_count": pass_count,
"blocked_count": blocked_count,
"secret_values_retained": False,
"database_role_live_verification_deferred_to_fail_closed_canary": True,
"production_repoint_executed": False,
}
rendered = json.dumps(receipt, indent=2, sort_keys=True) + "\n"
if args.output:
args.output.write_text(rendered, encoding="utf-8")
else:
print(rendered, end="")
return 0 if blocked_count == 0 else 1
if __name__ == "__main__":
raise SystemExit(main())