Add passwordless GCP operator and full Leo benchmark
This commit is contained in:
twentyOne2x
parent
1a0abd882d
commit
b1ff5a4629
11 changed files with 4538 additions and 0 deletions
196
.github/workflows/gcp-iap-operator.yml
vendored
Normal file
196
.github/workflows/gcp-iap-operator.yml
vendored
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
name: gcp-iap-operator
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
operation:
|
||||
description: Fixed reviewed operation
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- status
|
||||
- direct-claim-replay
|
||||
- cleanup-clone
|
||||
request_id:
|
||||
description: iap- followed by 12-32 lowercase letters or digits
|
||||
required: true
|
||||
type: string
|
||||
target_db:
|
||||
description: teleo_clone_status for status, otherwise the exact generated clone database
|
||||
required: true
|
||||
default: teleo_clone_status
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: gcp-iap-operator-${{ github.run_id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROJECT_ID: teleo-501523
|
||||
WORKLOAD_IDENTITY_PROVIDER: projects/785938879453/locations/global/workloadIdentityPools/github-actions/providers/teleo-iap-operator
|
||||
STATUS_SERVICE_ACCOUNT: sa-teleo-iap-status@teleo-501523.iam.gserviceaccount.com
|
||||
CLONE_SERVICE_ACCOUNT: sa-teleo-iap-clone-operator@teleo-501523.iam.gserviceaccount.com
|
||||
EXPECTED_WORKFLOW_REF: living-ip/teleo-infrastructure/.github/workflows/gcp-iap-operator.yml@refs/heads/main
|
||||
DISPATCHER_VERSION: teleo-gcp-iap-operator-v1
|
||||
BUNDLE_DIR: ${{ runner.temp }}/gcp-iap-operator-bundle
|
||||
RESULT_DIR: ${{ runner.temp }}/gcp-iap-operator-result
|
||||
|
||||
jobs:
|
||||
operate:
|
||||
name: Fixed IAP operator command
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Check out the dispatched main revision
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Validate fixed inputs and select identity
|
||||
id: validate
|
||||
shell: bash
|
||||
env:
|
||||
OPERATION: ${{ inputs.operation }}
|
||||
REQUEST_ID: ${{ inputs.request_id }}
|
||||
TARGET_DB: ${{ inputs.target_db }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${GITHUB_REF}" == "refs/heads/main" ]] || exit 2
|
||||
[[ "${GITHUB_WORKFLOW_REF}" == "${EXPECTED_WORKFLOW_REF}" ]] || exit 2
|
||||
request_id_re='^iap-[a-z0-9]{12,32}$'
|
||||
target_db_re='^teleo_clone_[a-z0-9][a-z0-9_]{0,50}$'
|
||||
[[ "${REQUEST_ID}" =~ ${request_id_re} ]] || exit 2
|
||||
[[ "${TARGET_DB}" =~ ${target_db_re} ]] || exit 2
|
||||
case "${OPERATION}" in
|
||||
status)
|
||||
[[ "${TARGET_DB}" == "teleo_clone_status" ]] || exit 2
|
||||
service_account="${STATUS_SERVICE_ACCOUNT}"
|
||||
;;
|
||||
direct-claim-replay|cleanup-clone)
|
||||
[[ "${TARGET_DB}" != "teleo_clone_status" ]] || exit 2
|
||||
service_account="${CLONE_SERVICE_ACCOUNT}"
|
||||
;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
printf 'service_account=%s\n' "${service_account}" >> "${GITHUB_OUTPUT}"
|
||||
install -d -m 0700 "${RESULT_DIR}"
|
||||
: > "${RESULT_DIR}/result.json"
|
||||
chmod 0600 "${RESULT_DIR}/result.json"
|
||||
|
||||
- name: Build strict reviewed operation bundle
|
||||
if: ${{ inputs.operation != 'status' }}
|
||||
shell: bash
|
||||
env:
|
||||
OPERATION: ${{ inputs.operation }}
|
||||
REQUEST_ID: ${{ inputs.request_id }}
|
||||
TARGET_DB: ${{ inputs.target_db }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
root = Path.cwd().resolve()
|
||||
operation = os.environ["OPERATION"]
|
||||
request_id = os.environ["REQUEST_ID"]
|
||||
target_db = os.environ["TARGET_DB"]
|
||||
common = ["scripts/gcp_iap_operator.sh"]
|
||||
direct_claim = [
|
||||
"scripts/run_gcp_generated_db_direct_claim_suite.py",
|
||||
"scripts/run_leo_clone_bound_handler_checkpoint.py",
|
||||
"scripts/working_leo_open_ended_benchmark.py",
|
||||
"hermes-agent/leoclean-bin/cloudsql_memory_tool.py",
|
||||
"ops/postgres_parity_manifest.sql",
|
||||
f"docs/reports/leo-working-state-20260709/{target_db}-canonical-parity-receipt.json",
|
||||
]
|
||||
allowed = common + (direct_claim if operation == "direct-claim-replay" else [])
|
||||
if operation not in {"direct-claim-replay", "cleanup-clone"}:
|
||||
raise SystemExit("bundle requested for a non-clone operation")
|
||||
head = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
if head != os.environ["GITHUB_SHA"]:
|
||||
raise SystemExit("checkout HEAD does not match GITHUB_SHA")
|
||||
files = {}
|
||||
payloads = {}
|
||||
for relative in allowed:
|
||||
subprocess.run(
|
||||
["git", "ls-files", "--error-unmatch", "--", relative],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
)
|
||||
path = root / relative
|
||||
if not path.is_file() or path.is_symlink() or root not in path.resolve().parents:
|
||||
raise SystemExit(f"unsafe or missing tracked bundle file: {relative}")
|
||||
data = path.read_bytes()
|
||||
payloads[relative] = data
|
||||
files[relative] = {"sha256": hashlib.sha256(data).hexdigest(), "size": len(data)}
|
||||
manifest = {
|
||||
"schema": "livingip.gcpIapOperatorBundle.v1",
|
||||
"dispatcher_version": os.environ["DISPATCHER_VERSION"],
|
||||
"operation": operation,
|
||||
"request_id": request_id,
|
||||
"target_db": target_db,
|
||||
"git_commit": head,
|
||||
"ref": os.environ["GITHUB_REF"],
|
||||
"workflow_ref": os.environ["GITHUB_WORKFLOW_REF"],
|
||||
"files": files,
|
||||
}
|
||||
manifest_bytes = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode()
|
||||
bundle_dir = Path(os.environ["BUNDLE_DIR"])
|
||||
bundle_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
os.chmod(bundle_dir, 0o700)
|
||||
bundle_path = bundle_dir / f"{request_id}.tar.gz"
|
||||
with bundle_path.open("xb") as raw:
|
||||
with gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as compressed:
|
||||
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive:
|
||||
for relative, data in sorted({**payloads, "bundle-manifest.json": manifest_bytes}.items()):
|
||||
info = tarfile.TarInfo(relative)
|
||||
info.size = len(data)
|
||||
info.mode = 0o600
|
||||
info.mtime = 0
|
||||
info.uid = info.gid = 0
|
||||
info.uname = info.gname = "root"
|
||||
archive.addfile(info, io.BytesIO(data))
|
||||
os.chmod(bundle_path, 0o600)
|
||||
PY
|
||||
|
||||
- name: Authenticate with short-lived workload identity federation
|
||||
id: auth
|
||||
uses: google-github-actions/auth@v3
|
||||
with:
|
||||
workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ steps.validate.outputs.service_account }}
|
||||
create_credentials_file: true
|
||||
cleanup_credentials: true
|
||||
export_environment_variables: true
|
||||
|
||||
- name: Install Google Cloud CLI
|
||||
uses: google-github-actions/setup-gcloud@v3
|
||||
with:
|
||||
project_id: ${{ env.PROJECT_ID }}
|
||||
|
||||
- name: Run fixed IAP operation
|
||||
shell: bash
|
||||
env:
|
||||
OPERATION: ${{ inputs.operation }}
|
||||
REQUEST_ID: ${{ inputs.request_id }}
|
||||
TARGET_DB: ${{ inputs.target_db }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
scripts/gcp_iap_operator.sh "${OPERATION}" "${REQUEST_ID}" "${TARGET_DB}"
|
||||
|
||||
- name: Upload sanitized result
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: gcp-iap-operator-${{ github.run_id }}
|
||||
path: ${{ env.RESULT_DIR }}/result.json
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -10,6 +10,7 @@ __pycache__/
|
|||
|
||||
# Secrets (never commit)
|
||||
secrets/
|
||||
gha-creds-*.json
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
|
|
|
|||
|
|
@ -130,6 +130,11 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c
|
|||
- Current GCP Leo runtime observability: `docs/reports/leo-working-state-20260709/gcp-leo-runtime-observability-current.md`
|
||||
- Current GCP Leo runtime observability JSON: `docs/reports/leo-working-state-20260709/gcp-leo-runtime-observability-current.json`
|
||||
- Prior auth-capable GCP blocker: `docs/reports/leo-working-state-20260709/gcp-db-parity-current-blocker-20260709.json`
|
||||
- PR #77 VPS auto-deploy/runtime non-change readback: `docs/reports/leo-working-state-20260709/pr77-vps-auto-deploy-runtime-nonchange-current.md`
|
||||
- Independent VPS source baseline for the GCP working-Leo suite: `docs/reports/leo-working-state-20260709/gcp-working-leo-source-baseline-20260712.json`
|
||||
- Disposable GCP clone canonical-parity receipt: `docs/reports/leo-working-state-20260709/teleo_clone_20260711t1816z-canonical-parity-receipt.json`
|
||||
- Passwordless GCP operator bootstrap plan: `ops/plan_gcp_iap_operator_access.py`
|
||||
- Passwordless fixed-operation GCP workflow: `.github/workflows/gcp-iap-operator.yml`
|
||||
|
||||
## Images
|
||||
|
||||
|
|
@ -190,3 +195,6 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c
|
|||
- A live Telegram regression canary should be rerun after any production apply; no production apply has been authorized or executed.
|
||||
- GCP control-plane Chrome readback now proves project `teleo-501523`, running VM `teleo-prod-1` in `europe-west6-a`, and one available private-only PostgreSQL `16.14` Cloud SQL instance, `teleo-pgvector-standby`, in `europe-west6-c`. Cloud SQL Studio is reachable but exposes no enabled existing IAM database principal and requires built-in password authentication; no credential was submitted and no SQL ran. Database-row/runtime/DC-01-through-DC-06 parity therefore remains unproven. The exact next gate is an already-existing authenticated Studio principal or a separately authorized IAM/database-user enablement window.
|
||||
- GCP Console observability now separately proves `teleo-prod-1` is running and that serial output through `2026-07-10T15:57:54Z` contains recurring successful `leoclean-cloudsql-memory-sync.service` cycles. The latest exact `leoclean-gcp-prod-parallel.service` lifecycle event in the retained serial buffer is a Hermes start on July 9 with no later exact stop/failure. Cloud Logging has `92` guest/platform records over seven days but zero matches for `leoclean`, `hermes`, `gateway`, `postgres`, or `teleo-kb`, and Ops Agent is absent. This is bounded unit-activity evidence, not current PID/restart/SHA/profile/model/auth/Telegram/DB-row parity; no-post Cory execution remains unsafe and was not attempted. The agent-created Console tab was closed and no GCP/DB/service/IAM/deploy/credential/Telegram mutation occurred.
|
||||
- PR #77 auto-deployed to the VPS checkout at merge `1a0abd882d00c1b58fe203f42de42d4eeea61cbf` without restarting Leo: the deploy journal says `No Python changes - services not restarted`, the gateway remained PID `2999690` with `NRestarts=0`, and the apply worker/timer remained disabled and inactive. This proves source synchronization and runtime non-change, not a GCP replay.
|
||||
- A passwordless GCP operator is implemented and locally validated but is not yet bootstrapped in project `teleo-501523`. It uses GitHub OIDC/Workload Identity Federation, separate status and clone-operation service accounts, IAP TCP forwarding, OS Login, five-minute SSH keys, and a root-owned dispatcher limited to `status`, `direct-claim-replay`, and `cleanup-clone`. It stores no Google password, local refresh token, service-account JSON key, or API key. One final authenticated GCP admin session is still required to install and verify the IAM/IAP/OS Login/dispatcher path; the old public `/32` rule must remain enabled until the workflow status artifact passes.
|
||||
- The broader GCP working-Leo runner now has a complete `20`-prompt catalog and requires `32` GCP-executable non-tautological composition checks plus lifecycle, role-separation, immutable-source, and secret-redaction gates. Local tests and validation-only execution pass, and an independent VPS source baseline is retained with counts `1837/4145/4670/4916/17/26`; live GCP execution remains unproven because the SHA-bound target-identity receipt, four separated database roles/credentials, and passwordless operator bootstrap are not yet present. This does not inherit or claim the VPS-only `34/34` result.
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
{"artifact": "gcp_working_leo_source_baseline", "captured_at_utc": "2026-07-12T02:57:31.103344+00:00", "gate_schema": {"apply_role_exists": true, "approval_table": null, "approve_function": null, "assert_function": null, "finish_function": null, "review_principals_table": null, "review_role_exists": false}, "guard_snapshot": {"counts": {"kb_stage.kb_proposals": 26, "public.claim_edges": 4916, "public.claim_evidence": 4670, "public.claims": 1837, "public.reasoning_tools": 17, "public.sources": 4145}, "database_identity": {"current_database": "teleo", "postmaster_started_at": "2026-07-03 00:17:20.873566+00", "system_identifier": "7649789040005668902", "transaction_read_only": "on"}, "rowset_md5": {"kb_stage.kb_proposals": "2b31c5fddc9fa3548cf1d7874b5f417a", "public.claim_edges": "7b78b7753e50eeafbcecefa588f668dc", "public.claim_evidence": "244387a16b6210ad3d89850251062bb2", "public.claims": "ea11d4b63a00406a0d7b2d4d4ade0123", "public.reasoning_tools": "fb0330410709f53e3815725cc8de6eaf", "public.sources": "8209986c34f9d84681459c7033e012a5"}}, "instance": "teleo-prod-1", "problems": [], "project": "teleo-501523", "status": "pass"}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
# PR #77 VPS Auto-Deploy Runtime Readback
|
||||
|
||||
Observed UTC: `2026-07-12T00:11:37Z`
|
||||
|
||||
## Result
|
||||
|
||||
The VPS auto-deploy timer fast-forwarded
|
||||
`/opt/teleo-eval/workspaces/deploy-infra` and
|
||||
`/opt/teleo-eval/.last-deploy-sha` from merge `3b534abd` to PR #77 merge
|
||||
`1a0abd882d00c1b58fe203f42de42d4eeea61cbf`.
|
||||
|
||||
This was a source-checkout synchronization, not a live Leo feature or process
|
||||
change:
|
||||
|
||||
- The auto-deploy journal says `No Python changes - services not restarted`.
|
||||
- The deploy checkout is clean on `main...github/main`.
|
||||
- `leoclean-gateway.service` remains active/running on PID `2999690`, with
|
||||
`NRestarts=0` and start time `2026-07-10T03:47:11Z`.
|
||||
- `teleo-kb-apply-worker.service` remains disabled, inactive, and dead, with
|
||||
no main process and no restart.
|
||||
- `teleo-kb-apply-worker.timer` remains disabled, inactive, and dead, with no
|
||||
last trigger.
|
||||
|
||||
## Auto-Deploy Journal
|
||||
|
||||
```text
|
||||
2026-07-12T00:08:03+00:00 New commits: 3b534abd -> 1a0abd88
|
||||
2026-07-12T00:08:10+00:00 Syntax check passed
|
||||
2026-07-12T00:08:11+00:00 Files synced
|
||||
2026-07-12T00:08:11+00:00 No Python changes - services not restarted
|
||||
2026-07-12T00:08:11+00:00 Deploy complete: 1a0abd8 Merge pull request #77 from living-ip/codex/gcp-canonical-clone-proof-20260711
|
||||
```
|
||||
|
||||
## Claim Ceiling
|
||||
|
||||
PR #77 is present in the VPS deploy checkout, but the live Leo gateway did not
|
||||
restart and the KB apply worker stayed disabled. This proves the hardening
|
||||
scripts did not change the currently running VPS Leo. It does not prove a
|
||||
future restart would preserve behavior, and it does not prove the hardened GCP
|
||||
replay until that replay runs against the disposable Cloud SQL database.
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
{
|
||||
"artifact": "canonical_postgres_parity_verification",
|
||||
"details": {
|
||||
"application_role_mismatches": {},
|
||||
"performance": [
|
||||
{
|
||||
"query": "claim_edges_by_claim_id",
|
||||
"source_ms": 0.079,
|
||||
"source_ratio": 0.3944,
|
||||
"target_ms": 1.972
|
||||
},
|
||||
{
|
||||
"query": "claim_evidence_by_claim_id",
|
||||
"source_ms": 0.072,
|
||||
"source_ratio": 0.2068,
|
||||
"target_ms": 1.034
|
||||
},
|
||||
{
|
||||
"query": "count_claims",
|
||||
"source_ms": 0.293,
|
||||
"source_ratio": 0.3056,
|
||||
"target_ms": 1.528
|
||||
},
|
||||
{
|
||||
"query": "pending_proposals_latest",
|
||||
"source_ms": 0.034,
|
||||
"source_ratio": 0.0062,
|
||||
"target_ms": 0.031
|
||||
}
|
||||
],
|
||||
"performance_problems": [],
|
||||
"required_extension_mismatches": {},
|
||||
"source_connection": {
|
||||
"database_collation": null,
|
||||
"database_ctype": null,
|
||||
"server_address": null,
|
||||
"server_port": null,
|
||||
"ssl": null,
|
||||
"ssl_version": null
|
||||
},
|
||||
"source_database": "teleo",
|
||||
"source_table_count": 39,
|
||||
"source_total_rows": 52164,
|
||||
"structural_hashes": {
|
||||
"columns": {
|
||||
"source": "50b502967aa85b88bd204c7b9035de60bf21a6a6973817e8b8121d0cd9faab81",
|
||||
"target": "50b502967aa85b88bd204c7b9035de60bf21a6a6973817e8b8121d0cd9faab81"
|
||||
},
|
||||
"constraints": {
|
||||
"source": "8b26d26fda56f09325e0603aae48051da08dfc6c35322fa88880acdabb8ad859",
|
||||
"target": "8b26d26fda56f09325e0603aae48051da08dfc6c35322fa88880acdabb8ad859"
|
||||
},
|
||||
"functions": {
|
||||
"source": "4bddee626b3ab62af236493001d2a09f2d56c084beb7e6c69cd082ce4aba155b",
|
||||
"target": "4bddee626b3ab62af236493001d2a09f2d56c084beb7e6c69cd082ce4aba155b"
|
||||
},
|
||||
"indexes": {
|
||||
"source": "5c4f7567e9ad0f25da0ab15f650a1005397ae9c727af1ab41e2675c46a5ebce5",
|
||||
"target": "5c4f7567e9ad0f25da0ab15f650a1005397ae9c727af1ab41e2675c46a5ebce5"
|
||||
},
|
||||
"policies": {
|
||||
"source": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945",
|
||||
"target": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"
|
||||
},
|
||||
"schemas": {
|
||||
"source": "2fdf551b403db2d79813c8fd2869dadfeba261fac2aaa718b355cd7414640979",
|
||||
"target": "2fdf551b403db2d79813c8fd2869dadfeba261fac2aaa718b355cd7414640979"
|
||||
},
|
||||
"sequences": {
|
||||
"source": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945",
|
||||
"target": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"
|
||||
},
|
||||
"triggers": {
|
||||
"source": "4d5f3394480c8410035dac5457e144c7db80fa0820ff49beb93aa819a7282c64",
|
||||
"target": "4d5f3394480c8410035dac5457e144c7db80fa0820ff49beb93aa819a7282c64"
|
||||
},
|
||||
"types": {
|
||||
"source": "365e82ac205319d25d14c8b16026978f0e6023ae0c58767f700d7f2ea659b70b",
|
||||
"target": "365e82ac205319d25d14c8b16026978f0e6023ae0c58767f700d7f2ea659b70b"
|
||||
},
|
||||
"views": {
|
||||
"source": "5c33fa260f25dccffc214f795c66e3dbae89d91d0dbf8a98122ace1dcda46799",
|
||||
"target": "5c33fa260f25dccffc214f795c66e3dbae89d91d0dbf8a98122ace1dcda46799"
|
||||
}
|
||||
},
|
||||
"table_mismatches": [],
|
||||
"target_connection": {
|
||||
"database_collation": "en_US.UTF8",
|
||||
"database_ctype": "en_US.UTF8",
|
||||
"server_address": "10.61.0.3",
|
||||
"server_port": 5432,
|
||||
"ssl": true,
|
||||
"ssl_version": "TLSv1.3"
|
||||
},
|
||||
"target_database": "teleo_clone_20260711t1816z",
|
||||
"target_extra_application_roles": [],
|
||||
"target_table_count": 39,
|
||||
"target_total_rows": 52164
|
||||
},
|
||||
"error": null,
|
||||
"generated_at_utc": "2026-07-11T18:25:05.711214+00:00",
|
||||
"not_proven_by_this_artifact": [
|
||||
"GCP staging Leo composition replay",
|
||||
"ongoing replication after the manifest timestamps",
|
||||
"production cutover"
|
||||
],
|
||||
"private_connectivity": {
|
||||
"required": false,
|
||||
"status": "not_applicable_local_restore"
|
||||
},
|
||||
"problems": [],
|
||||
"scope": "local_restore",
|
||||
"source_manifest": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/outputs/gcp-canonical-exported-snapshot-20260711T0727Z/source-manifest.jsonl",
|
||||
"status": "pass",
|
||||
"target_manifest": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/outputs/gcp-cloudsql-canonical-clone-20260711T1816Z/target-manifest-role-parity.jsonl"
|
||||
}
|
||||
752
ops/plan_gcp_iap_operator_access.py
Executable file
752
ops/plan_gcp_iap_operator_access.py
Executable file
|
|
@ -0,0 +1,752 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Emit the dry-run bootstrap and revocation plan for GitHub-to-GCP IAP SSH."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shlex
|
||||
|
||||
PROJECT = "teleo-501523"
|
||||
PROJECT_NUMBER = "785938879453"
|
||||
GITHUB_REPOSITORY = "living-ip/teleo-infrastructure"
|
||||
WORKFLOW_PATH = ".github/workflows/gcp-iap-operator.yml"
|
||||
WORKFLOW_REF = f"{GITHUB_REPOSITORY}/{WORKFLOW_PATH}@refs/heads/main"
|
||||
POOL = "github-actions"
|
||||
PROVIDER = "teleo-iap-operator"
|
||||
PROVIDER_RESOURCE = f"projects/{PROJECT_NUMBER}/locations/global/workloadIdentityPools/{POOL}/providers/{PROVIDER}"
|
||||
|
||||
ZONE = "europe-west6-a"
|
||||
INSTANCE = "teleo-prod-1"
|
||||
INSTANCE_INTERNAL_IP = "10.60.0.3"
|
||||
NETWORK = "teleo-staging-net"
|
||||
TARGET_TAG = "teleo-prod-ssh"
|
||||
VM_SERVICE_ACCOUNT = f"sa-teleo-prod-vm@{PROJECT}.iam.gserviceaccount.com"
|
||||
STATUS_SERVICE_ACCOUNT_ID = "sa-teleo-iap-status"
|
||||
STATUS_SERVICE_ACCOUNT = f"{STATUS_SERVICE_ACCOUNT_ID}@{PROJECT}.iam.gserviceaccount.com"
|
||||
CLONE_SERVICE_ACCOUNT_ID = "sa-teleo-iap-clone-operator"
|
||||
CLONE_SERVICE_ACCOUNT = f"{CLONE_SERVICE_ACCOUNT_ID}@{PROJECT}.iam.gserviceaccount.com"
|
||||
|
||||
CUSTOM_ROLE_ID = "teleoIapSshDiscovery"
|
||||
CUSTOM_ROLE = f"projects/{PROJECT}/roles/{CUSTOM_ROLE_ID}"
|
||||
CUSTOM_PERMISSIONS = [
|
||||
"compute.instances.get",
|
||||
"compute.instances.list",
|
||||
"compute.projects.get",
|
||||
]
|
||||
IAP_ROLE = "roles/iap.tunnelResourceAccessor"
|
||||
IAP_CONDITION = f'destination.ip == "{INSTANCE_INTERNAL_IP}" && destination.port == 22'
|
||||
PROVIDER_CONDITION = " && ".join(
|
||||
[
|
||||
f"assertion.repository == '{GITHUB_REPOSITORY}'",
|
||||
"assertion.ref == 'refs/heads/main'",
|
||||
f"assertion.workflow_ref == '{WORKFLOW_REF}'",
|
||||
"assertion.event_name == 'workflow_dispatch'",
|
||||
]
|
||||
)
|
||||
ATTRIBUTE_MAPPING = ",".join(
|
||||
[
|
||||
"google.subject=assertion.sub",
|
||||
"attribute.repository=assertion.repository",
|
||||
"attribute.ref=assertion.ref",
|
||||
"attribute.workflow_ref=assertion.workflow_ref",
|
||||
"attribute.event_name=assertion.event_name",
|
||||
]
|
||||
)
|
||||
WIF_PRINCIPAL = (
|
||||
f"principal://iam.googleapis.com/projects/{PROJECT_NUMBER}/locations/global/"
|
||||
f"workloadIdentityPools/{POOL}/subject/repo:{GITHUB_REPOSITORY}:ref:refs/heads/main"
|
||||
)
|
||||
|
||||
IAP_FIREWALL_RULE = "teleo-prod-allow-ssh-iap"
|
||||
CHANGING_PUBLIC_RULE = "teleo-prod-allow-ssh-current-ip"
|
||||
IAP_SOURCE_RANGE = "35.235.240.0/20"
|
||||
DISPATCHER_VERSION = "teleo-gcp-iap-operator-v1"
|
||||
|
||||
FORBIDDEN_OPERATOR_ROLES = [
|
||||
"roles/compute.admin",
|
||||
"roles/secretmanager.admin",
|
||||
"roles/secretmanager.secretAccessor",
|
||||
"roles/cloudsql.admin",
|
||||
"roles/artifactregistry.admin",
|
||||
"roles/artifactregistry.writer",
|
||||
]
|
||||
|
||||
|
||||
def shell_join(parts: list[str]) -> str:
|
||||
return " ".join(shlex.quote(part) for part in parts)
|
||||
|
||||
|
||||
def service_account_member(email: str) -> str:
|
||||
return f"serviceAccount:{email}"
|
||||
|
||||
|
||||
def bootstrap_commands() -> list[str]:
|
||||
accounts = [
|
||||
(STATUS_SERVICE_ACCOUNT_ID, "Teleo IAP status canary"),
|
||||
(CLONE_SERVICE_ACCOUNT_ID, "Teleo IAP clone operator"),
|
||||
]
|
||||
commands = [
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"services",
|
||||
"enable",
|
||||
"compute.googleapis.com",
|
||||
"iap.googleapis.com",
|
||||
"iamcredentials.googleapis.com",
|
||||
"sts.googleapis.com",
|
||||
"--project",
|
||||
PROJECT,
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"workload-identity-pools",
|
||||
"describe",
|
||||
POOL,
|
||||
"--location",
|
||||
"global",
|
||||
"--project",
|
||||
PROJECT,
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"workload-identity-pools",
|
||||
"providers",
|
||||
"create-oidc",
|
||||
PROVIDER,
|
||||
"--location",
|
||||
"global",
|
||||
"--workload-identity-pool",
|
||||
POOL,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--issuer-uri",
|
||||
"https://token.actions.githubusercontent.com",
|
||||
"--attribute-mapping",
|
||||
ATTRIBUTE_MAPPING,
|
||||
"--attribute-condition",
|
||||
PROVIDER_CONDITION,
|
||||
"--display-name",
|
||||
"Teleo fixed IAP operator workflow",
|
||||
]
|
||||
),
|
||||
*[
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"service-accounts",
|
||||
"create",
|
||||
account_id,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--display-name",
|
||||
display_name,
|
||||
]
|
||||
)
|
||||
for account_id, display_name in accounts
|
||||
],
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"roles",
|
||||
"create",
|
||||
CUSTOM_ROLE_ID,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--title",
|
||||
"Teleo exact-instance SSH discovery",
|
||||
"--description",
|
||||
"Only the instance and project reads required by gcloud compute ssh",
|
||||
"--permissions",
|
||||
",".join(CUSTOM_PERMISSIONS),
|
||||
"--stage",
|
||||
"GA",
|
||||
]
|
||||
),
|
||||
]
|
||||
|
||||
for account in (STATUS_SERVICE_ACCOUNT, CLONE_SERVICE_ACCOUNT):
|
||||
member = service_account_member(account)
|
||||
commands.extend(
|
||||
[
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"service-accounts",
|
||||
"add-iam-policy-binding",
|
||||
account,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--member",
|
||||
WIF_PRINCIPAL,
|
||||
"--role",
|
||||
"roles/iam.workloadIdentityUser",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"projects",
|
||||
"add-iam-policy-binding",
|
||||
PROJECT,
|
||||
"--member",
|
||||
member,
|
||||
"--role",
|
||||
CUSTOM_ROLE,
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"projects",
|
||||
"add-iam-policy-binding",
|
||||
PROJECT,
|
||||
"--member",
|
||||
member,
|
||||
"--role",
|
||||
IAP_ROLE,
|
||||
"--condition",
|
||||
f"expression={IAP_CONDITION},title=teleo-prod-1-ssh-only",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"service-accounts",
|
||||
"add-iam-policy-binding",
|
||||
VM_SERVICE_ACCOUNT,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--member",
|
||||
member,
|
||||
"--role",
|
||||
"roles/iam.serviceAccountUser",
|
||||
]
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
commands.extend(
|
||||
[
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"compute",
|
||||
"instances",
|
||||
"add-iam-policy-binding",
|
||||
INSTANCE,
|
||||
"--zone",
|
||||
ZONE,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--member",
|
||||
service_account_member(STATUS_SERVICE_ACCOUNT),
|
||||
"--role",
|
||||
"roles/compute.osLogin",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"compute",
|
||||
"instances",
|
||||
"add-iam-policy-binding",
|
||||
INSTANCE,
|
||||
"--zone",
|
||||
ZONE,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--member",
|
||||
service_account_member(CLONE_SERVICE_ACCOUNT),
|
||||
"--role",
|
||||
"roles/compute.osAdminLogin",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"compute",
|
||||
"instances",
|
||||
"add-metadata",
|
||||
INSTANCE,
|
||||
"--zone",
|
||||
ZONE,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--metadata",
|
||||
"enable-oslogin=TRUE",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"compute",
|
||||
"firewall-rules",
|
||||
"create",
|
||||
IAP_FIREWALL_RULE,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--network",
|
||||
NETWORK,
|
||||
"--direction",
|
||||
"INGRESS",
|
||||
"--action",
|
||||
"ALLOW",
|
||||
"--rules",
|
||||
"tcp:22",
|
||||
"--source-ranges",
|
||||
IAP_SOURCE_RANGE,
|
||||
"--target-tags",
|
||||
TARGET_TAG,
|
||||
"--enable-logging",
|
||||
]
|
||||
),
|
||||
"# Install the reviewed dispatcher through the existing admin path before removing that path.",
|
||||
'BOOTSTRAP_SSH_DIR=$(mktemp -d "${TMPDIR:-/tmp}/teleo-iap-bootstrap.XXXXXX"); '
|
||||
'chmod 0700 "${BOOTSTRAP_SSH_DIR}"; '
|
||||
'BOOTSTRAP_SSH_KEY="${BOOTSTRAP_SSH_DIR}/ssh_key"; '
|
||||
"trap 'rm -rf \"${BOOTSTRAP_SSH_DIR}\"' EXIT",
|
||||
"install -m 0600 scripts/gcp_iap_operator.sh "
|
||||
'"${BOOTSTRAP_SSH_DIR}/gcp_iap_operator.sh"; '
|
||||
'(cd "${BOOTSTRAP_SSH_DIR}" && sha256sum gcp_iap_operator.sh > dispatcher.sha256); '
|
||||
'chmod 0600 "${BOOTSTRAP_SSH_DIR}/dispatcher.sha256"',
|
||||
"gcloud compute scp "
|
||||
'"${BOOTSTRAP_SSH_DIR}/gcp_iap_operator.sh" '
|
||||
'"${BOOTSTRAP_SSH_DIR}/dispatcher.sha256" '
|
||||
f"{INSTANCE}:/tmp/ --zone {ZONE} --project {PROJECT} "
|
||||
'--tunnel-through-iap --ssh-key-file="${BOOTSTRAP_SSH_KEY}" '
|
||||
'--ssh-key-expire-after=5m',
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"compute",
|
||||
"ssh",
|
||||
INSTANCE,
|
||||
"--zone",
|
||||
ZONE,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--command",
|
||||
"cd /tmp && sha256sum -c dispatcher.sha256 && "
|
||||
f"grep -F 'readonly DISPATCHER_VERSION=\"{DISPATCHER_VERSION}\"' gcp_iap_operator.sh && "
|
||||
"sudo install -o root -g root -m 0755 /tmp/gcp_iap_operator.sh "
|
||||
"/usr/local/sbin/teleo-gcp-iap-operator && "
|
||||
"test \"$(sudo sha256sum /usr/local/sbin/teleo-gcp-iap-operator | awk '{print $1}')\" "
|
||||
"= \"$(awk '{print $1}' /tmp/dispatcher.sha256)\" && "
|
||||
"sudo install -d -o root -g root -m 0700 /var/lib/teleo-gcp-iap-operator/runs && "
|
||||
"rm -f /tmp/gcp_iap_operator.sh /tmp/dispatcher.sha256",
|
||||
]
|
||||
)
|
||||
+ ' --tunnel-through-iap --ssh-key-file="${BOOTSTRAP_SSH_KEY}" '
|
||||
'--ssh-key-expire-after=5m',
|
||||
]
|
||||
)
|
||||
return commands
|
||||
|
||||
|
||||
def status_verification_commands() -> list[str]:
|
||||
request_id = "iap-bootstrapstatus01"
|
||||
return [
|
||||
shell_join(
|
||||
[
|
||||
"gh",
|
||||
"workflow",
|
||||
"run",
|
||||
"gcp-iap-operator.yml",
|
||||
"--repo",
|
||||
GITHUB_REPOSITORY,
|
||||
"--ref",
|
||||
"main",
|
||||
"-f",
|
||||
"operation=status",
|
||||
"-f",
|
||||
f"request_id={request_id}",
|
||||
"-f",
|
||||
"target_db=teleo_clone_status",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gh",
|
||||
"run",
|
||||
"list",
|
||||
"--repo",
|
||||
GITHUB_REPOSITORY,
|
||||
"--workflow",
|
||||
"gcp-iap-operator.yml",
|
||||
"--event",
|
||||
"workflow_dispatch",
|
||||
"--limit",
|
||||
"1",
|
||||
]
|
||||
),
|
||||
"IAP_RUN_ID=$(gh run list --repo "
|
||||
f"{GITHUB_REPOSITORY} --workflow gcp-iap-operator.yml --event workflow_dispatch --limit 1 "
|
||||
"--json databaseId --jq '.[0].databaseId'); test -n \"${IAP_RUN_ID}\"",
|
||||
'IAP_VERIFY_DIR=$(mktemp -d "${TMPDIR:-/tmp}/teleo-iap-verify.XXXXXX"); '
|
||||
'chmod 0700 "${IAP_VERIFY_DIR}"; trap \'rm -rf "${IAP_VERIFY_DIR}"\' EXIT',
|
||||
'gh run watch "${IAP_RUN_ID}" --repo ' + GITHUB_REPOSITORY + " --exit-status",
|
||||
'gh run download "${IAP_RUN_ID}" --repo '
|
||||
+ GITHUB_REPOSITORY
|
||||
+ ' --name "gcp-iap-operator-${IAP_RUN_ID}" --dir "${IAP_VERIFY_DIR}"',
|
||||
"python3 -c 'import json,sys; p=json.load(open(sys.argv[1])); "
|
||||
"import hashlib,pathlib; "
|
||||
'r=p.get("remote_result") or {}; assert p.get("status")=="pass"; '
|
||||
f'assert p.get("request_id")=="{request_id}"; '
|
||||
'assert p.get("operation")=="status"; '
|
||||
f'assert r.get("expected_internal_ip")=="{INSTANCE_INTERNAL_IP}"; '
|
||||
f'assert r.get("dispatcher_version")=="{DISPATCHER_VERSION}"; '
|
||||
'assert r.get("dispatcher_sha256")==hashlib.sha256('
|
||||
'pathlib.Path("scripts/gcp_iap_operator.sh").read_bytes()).hexdigest(); '
|
||||
'assert r.get("active_state")=="active"\' "${IAP_VERIFY_DIR}/result.json"',
|
||||
"# The old public /32 rule may be disabled only after every command above succeeds.",
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"compute",
|
||||
"firewall-rules",
|
||||
"update",
|
||||
CHANGING_PUBLIC_RULE,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--disabled",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"compute",
|
||||
"firewall-rules",
|
||||
"describe",
|
||||
CHANGING_PUBLIC_RULE,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--format=value(disabled,sourceRanges)",
|
||||
]
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def revocation_commands() -> list[str]:
|
||||
commands = [
|
||||
"# Remove the remote dispatcher while IAP access still exists, then revoke cloud access.",
|
||||
'REVOKE_SSH_DIR=$(mktemp -d "${TMPDIR:-/tmp}/teleo-iap-revoke.XXXXXX"); '
|
||||
'chmod 0700 "${REVOKE_SSH_DIR}"; REVOKE_SSH_KEY="${REVOKE_SSH_DIR}/ssh_key"; '
|
||||
"trap 'rm -rf \"${REVOKE_SSH_DIR}\"' EXIT",
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"compute",
|
||||
"ssh",
|
||||
INSTANCE,
|
||||
"--zone",
|
||||
ZONE,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--tunnel-through-iap",
|
||||
"--command",
|
||||
"sudo rm -f /usr/local/sbin/teleo-gcp-iap-operator",
|
||||
]
|
||||
)
|
||||
+ ' --ssh-key-file="${REVOKE_SSH_KEY}" --ssh-key-expire-after=5m',
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"workload-identity-pools",
|
||||
"providers",
|
||||
"update-oidc",
|
||||
PROVIDER,
|
||||
"--location",
|
||||
"global",
|
||||
"--workload-identity-pool",
|
||||
POOL,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--disabled",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"compute",
|
||||
"firewall-rules",
|
||||
"delete",
|
||||
IAP_FIREWALL_RULE,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--quiet",
|
||||
]
|
||||
),
|
||||
]
|
||||
for account, os_role in (
|
||||
(STATUS_SERVICE_ACCOUNT, "roles/compute.osLogin"),
|
||||
(CLONE_SERVICE_ACCOUNT, "roles/compute.osAdminLogin"),
|
||||
):
|
||||
member = service_account_member(account)
|
||||
commands.extend(
|
||||
[
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"compute",
|
||||
"instances",
|
||||
"remove-iam-policy-binding",
|
||||
INSTANCE,
|
||||
"--zone",
|
||||
ZONE,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--member",
|
||||
member,
|
||||
"--role",
|
||||
os_role,
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"projects",
|
||||
"remove-iam-policy-binding",
|
||||
PROJECT,
|
||||
"--member",
|
||||
member,
|
||||
"--role",
|
||||
CUSTOM_ROLE,
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"projects",
|
||||
"remove-iam-policy-binding",
|
||||
PROJECT,
|
||||
"--member",
|
||||
member,
|
||||
"--role",
|
||||
IAP_ROLE,
|
||||
"--condition",
|
||||
f"expression={IAP_CONDITION},title=teleo-prod-1-ssh-only",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"service-accounts",
|
||||
"remove-iam-policy-binding",
|
||||
VM_SERVICE_ACCOUNT,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--member",
|
||||
member,
|
||||
"--role",
|
||||
"roles/iam.serviceAccountUser",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"service-accounts",
|
||||
"delete",
|
||||
account,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--quiet",
|
||||
]
|
||||
),
|
||||
]
|
||||
)
|
||||
commands.extend(
|
||||
[
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"workload-identity-pools",
|
||||
"providers",
|
||||
"delete",
|
||||
PROVIDER,
|
||||
"--location",
|
||||
"global",
|
||||
"--workload-identity-pool",
|
||||
POOL,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--quiet",
|
||||
]
|
||||
),
|
||||
shell_join(
|
||||
[
|
||||
"gcloud",
|
||||
"iam",
|
||||
"roles",
|
||||
"delete",
|
||||
CUSTOM_ROLE_ID,
|
||||
"--project",
|
||||
PROJECT,
|
||||
"--quiet",
|
||||
]
|
||||
),
|
||||
]
|
||||
)
|
||||
return commands
|
||||
|
||||
|
||||
def build_plan() -> dict[str, object]:
|
||||
return {
|
||||
"artifact": "teleo_gcp_iap_operator_access_plan",
|
||||
"mode": "dry_run_emit_only",
|
||||
"executes_commands": False,
|
||||
"project": PROJECT,
|
||||
"target": {
|
||||
"instance": INSTANCE,
|
||||
"zone": ZONE,
|
||||
"internal_ip": INSTANCE_INTERNAL_IP,
|
||||
"network": NETWORK,
|
||||
"target_tag": TARGET_TAG,
|
||||
},
|
||||
"github": {
|
||||
"repository": GITHUB_REPOSITORY,
|
||||
"ref": "refs/heads/main",
|
||||
"workflow_ref": WORKFLOW_REF,
|
||||
"event_name": "workflow_dispatch",
|
||||
},
|
||||
"wif": {
|
||||
"provider": PROVIDER_RESOURCE,
|
||||
"provider_condition": PROVIDER_CONDITION,
|
||||
"attribute_mapping": ATTRIBUTE_MAPPING,
|
||||
"principal": WIF_PRINCIPAL,
|
||||
"long_lived_google_credentials": False,
|
||||
},
|
||||
"identities": {
|
||||
"status": {
|
||||
"service_account": STATUS_SERVICE_ACCOUNT,
|
||||
"os_login_role": "roles/compute.osLogin",
|
||||
"sudo": False,
|
||||
"operations": ["status"],
|
||||
},
|
||||
"clone_operator": {
|
||||
"service_account": CLONE_SERVICE_ACCOUNT,
|
||||
"os_login_role": "roles/compute.osAdminLogin",
|
||||
"sudo": True,
|
||||
"operations": ["direct-claim-replay", "cleanup-clone"],
|
||||
"blast_radius": (
|
||||
"OS Admin can become root on teleo-prod-1. The identity is separated from status, "
|
||||
"the provider is pinned to the main workflow_dispatch workflow, and the checked-in "
|
||||
"workflow invokes only the root-owned fixed-operation dispatcher. Branch protection "
|
||||
"and workflow review remain required controls because OIDC does not attest file content."
|
||||
),
|
||||
},
|
||||
},
|
||||
"gcloud_ssh_custom_role": {
|
||||
"role": CUSTOM_ROLE,
|
||||
"permissions": CUSTOM_PERMISSIONS,
|
||||
"binding_scope": f"projects/{PROJECT}",
|
||||
"rationale": (
|
||||
"The official gcloud IAP SSH permission table requires instance get/list plus project lookup. "
|
||||
"This project-level role is read-only; exact destination authorization remains restricted by "
|
||||
"the IAP destination condition and instance-level OS Login binding."
|
||||
),
|
||||
},
|
||||
"iap": {
|
||||
"role": IAP_ROLE,
|
||||
"condition": IAP_CONDITION,
|
||||
"firewall": {
|
||||
"rule": IAP_FIREWALL_RULE,
|
||||
"source_range": IAP_SOURCE_RANGE,
|
||||
"protocol_ports": "tcp:22",
|
||||
"target_tag": TARGET_TAG,
|
||||
},
|
||||
},
|
||||
"service_account_user": {
|
||||
"target_service_account": VM_SERVICE_ACCOUNT,
|
||||
"role": "roles/iam.serviceAccountUser",
|
||||
"rationale": "Required to connect to a VM that runs as the dedicated VM service account.",
|
||||
},
|
||||
"explicitly_absent": {
|
||||
"roles": FORBIDDEN_OPERATOR_ROLES,
|
||||
"permissions": [
|
||||
"compute.instances.setMetadata",
|
||||
"compute.projects.setCommonInstanceMetadata",
|
||||
"secretmanager.versions.access",
|
||||
"cloudsql.instances.update",
|
||||
"artifactregistry.repositories.uploadArtifacts",
|
||||
],
|
||||
"credentials": [
|
||||
"Google password",
|
||||
"local user refresh token",
|
||||
"service-account JSON key",
|
||||
"API key",
|
||||
],
|
||||
},
|
||||
"bootstrap_commands": bootstrap_commands(),
|
||||
"post_bootstrap_verification": status_verification_commands(),
|
||||
"public_ingress_cutover": {
|
||||
"old_rule": CHANGING_PUBLIC_RULE,
|
||||
"disable_only_after": "successful workflow status artifact independently inspected",
|
||||
"automatic_fallback_to_public_ssh": False,
|
||||
},
|
||||
"audit_logging": {
|
||||
"guidance": (
|
||||
"Before enabling the operator workflow, merge project IAM auditConfigs rather than replacing "
|
||||
"the policy. Enable DATA_READ for iap.googleapis.com and DATA_WRITE for iap.googleapis.com, "
|
||||
"plus ADMIN_READ for "
|
||||
"compute.googleapis.com and iam.googleapis.com; retain IAP TCP tunnel and OS Login audit "
|
||||
"entries, firewall-rule logs, "
|
||||
"GitHub run ID, request_id, target_db, and sanitized result SHA-256. Never log auth outputs."
|
||||
),
|
||||
"readback_commands": [
|
||||
f"gcloud projects get-iam-policy {PROJECT} --format=json",
|
||||
(
|
||||
"gcloud logging read "
|
||||
'\'protoPayload.serviceName="iap.googleapis.com" AND '
|
||||
f'resource.labels.project_id="{PROJECT}"\' --project {PROJECT} --limit 20 --format=json'
|
||||
),
|
||||
(
|
||||
"gcloud logging read "
|
||||
f'\'resource.type="gce_firewall_rule" AND resource.labels.firewall_rule_id="{IAP_FIREWALL_RULE}"\' '
|
||||
f"--project {PROJECT} --limit 20 --format=json"
|
||||
),
|
||||
],
|
||||
},
|
||||
"independent_verification": [
|
||||
"Use an auditor identity that cannot impersonate either operator service account.",
|
||||
"Confirm the provider condition, both service-account policies, instance IAM policy, IAP condition, and firewall source/tag.",
|
||||
"Dispatch status from gh on main, inspect the sanitized artifact, and confirm no public /32 rule was disabled before that pass.",
|
||||
"Confirm the workflow created no service-account key and the runner removed its five-minute SSH key and auth credential file.",
|
||||
],
|
||||
"revocation_commands": revocation_commands(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--format", choices=("json", "shell"), default="json")
|
||||
args = parser.parse_args()
|
||||
plan = build_plan()
|
||||
if args.format == "shell":
|
||||
print("# DRY RUN: review these commands; this program never executes them.")
|
||||
for section in ("bootstrap_commands", "post_bootstrap_verification", "revocation_commands"):
|
||||
print(f"\n# {section}")
|
||||
for command in plan[section]:
|
||||
print(command)
|
||||
else:
|
||||
print(json.dumps(plan, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
603
scripts/gcp_iap_operator.sh
Executable file
603
scripts/gcp_iap_operator.sh
Executable file
|
|
@ -0,0 +1,603 @@
|
|||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
readonly DISPATCHER_VERSION="teleo-gcp-iap-operator-v1"
|
||||
readonly EXPECTED_WORKFLOW_REF="living-ip/teleo-infrastructure/.github/workflows/gcp-iap-operator.yml@refs/heads/main"
|
||||
readonly PROJECT_ID="teleo-501523"
|
||||
readonly INSTANCE="teleo-prod-1"
|
||||
readonly ZONE="europe-west6-a"
|
||||
readonly INSTANCE_INTERNAL_IP="10.60.0.3"
|
||||
readonly CLOUDSQL_HOST="10.61.0.3"
|
||||
readonly CLOUDSQL_SECRET="gcp-teleo-pgvector-standby-postgres-password"
|
||||
readonly REMOTE_DISPATCHER="/usr/local/sbin/teleo-gcp-iap-operator"
|
||||
readonly REMOTE_RUN_ROOT="/var/lib/teleo-gcp-iap-operator/runs"
|
||||
readonly LIVE_PROFILE="/home/teleo/.hermes/profiles/leoclean"
|
||||
readonly LIVE_SERVICE="leoclean-gcp-prod-parallel.service"
|
||||
readonly STATUS_TARGET="teleo_clone_status"
|
||||
readonly REQUEST_ID_RE='^iap-[a-z0-9]{12,32}$'
|
||||
readonly TARGET_DB_RE='^teleo_clone_[a-z0-9][a-z0-9_]{0,50}$'
|
||||
|
||||
RUNNER_TEMP_DIR=""
|
||||
VALIDATED_BUNDLE_DIR=""
|
||||
VALIDATED_BUNDLE_SHA256=""
|
||||
VALIDATED_BUNDLE_COMMIT=""
|
||||
|
||||
die() {
|
||||
printf 'gcp_iap_operator: %s\n' "$1" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
cleanup_runner_temp() {
|
||||
if [[ -n "$RUNNER_TEMP_DIR" && -d "$RUNNER_TEMP_DIR" && "$RUNNER_TEMP_DIR" == "${RUNNER_TEMP:-}/"* ]]; then
|
||||
rm -rf -- "$RUNNER_TEMP_DIR"
|
||||
fi
|
||||
}
|
||||
|
||||
validate_inputs() {
|
||||
local operation="$1"
|
||||
local request_id="$2"
|
||||
local target_db="$3"
|
||||
|
||||
case "$operation" in
|
||||
status | direct-claim-replay | cleanup-clone) ;;
|
||||
*) die "operation is not allowlisted" ;;
|
||||
esac
|
||||
[[ "$request_id" =~ $REQUEST_ID_RE ]] || die "request_id does not match the required format"
|
||||
[[ "$target_db" =~ $TARGET_DB_RE ]] || die "target_db must be a bounded teleo_clone_* identifier"
|
||||
if [[ "$operation" == "status" && "$target_db" != "$STATUS_TARGET" ]]; then
|
||||
die "status requires the fixed teleo_clone_status sentinel"
|
||||
fi
|
||||
if [[ "$operation" != "status" && "$target_db" == "$STATUS_TARGET" ]]; then
|
||||
die "clone operations cannot use the status sentinel"
|
||||
fi
|
||||
}
|
||||
|
||||
emit_json() {
|
||||
local operation="$1"
|
||||
local request_id="$2"
|
||||
local target_db="$3"
|
||||
local status="$4"
|
||||
shift 4
|
||||
python3 - "$operation" "$request_id" "$target_db" "$status" "$@" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
operation, request_id, target_db, status, *pairs = sys.argv[1:]
|
||||
payload = {
|
||||
"schema": "livingip.gcpIapOperatorRemoteResult.v1",
|
||||
"operation": operation,
|
||||
"request_id": request_id,
|
||||
"target_db": target_db,
|
||||
"status": status,
|
||||
}
|
||||
for pair in pairs:
|
||||
key, separator, value = pair.partition("=")
|
||||
if not separator:
|
||||
raise SystemExit("invalid result field")
|
||||
if key in {"nrestarts", "result_bytes"}:
|
||||
payload[key] = int(value)
|
||||
elif key in {"no_message_send", "live_service_unchanged", "live_profile_unchanged"}:
|
||||
payload[key] = value == "true"
|
||||
else:
|
||||
payload[key] = value
|
||||
print(json.dumps(payload, sort_keys=True))
|
||||
PY
|
||||
}
|
||||
|
||||
remote_status() {
|
||||
local request_id="$1"
|
||||
local target_db="$2"
|
||||
local active_state sub_state nrestarts dispatcher_hash dispatcher_state
|
||||
|
||||
[[ -x "$REMOTE_DISPATCHER" && ! -L "$REMOTE_DISPATCHER" ]] || die "root-owned remote dispatcher is absent"
|
||||
dispatcher_state="$(stat -c '%U:%G:%a' "$REMOTE_DISPATCHER")"
|
||||
[[ "$dispatcher_state" == "root:root:755" ]] || die "remote dispatcher ownership or mode drifted"
|
||||
dispatcher_hash="$(sha256sum "$REMOTE_DISPATCHER" | awk '{print $1}')"
|
||||
[[ "$dispatcher_hash" =~ ^[0-9a-f]{64}$ ]] || die "remote dispatcher hash readback was invalid"
|
||||
active_state="$(systemctl show "$LIVE_SERVICE" --property=ActiveState --value)"
|
||||
sub_state="$(systemctl show "$LIVE_SERVICE" --property=SubState --value)"
|
||||
nrestarts="$(systemctl show "$LIVE_SERVICE" --property=NRestarts --value)"
|
||||
[[ "$active_state" =~ ^[a-z]+$ ]] || die "service ActiveState readback was invalid"
|
||||
[[ "$sub_state" =~ ^[a-z-]+$ ]] || die "service SubState readback was invalid"
|
||||
[[ "$nrestarts" =~ ^[0-9]+$ ]] || die "service NRestarts readback was invalid"
|
||||
|
||||
emit_json \
|
||||
"status" "$request_id" "$target_db" "pass" \
|
||||
"instance=$INSTANCE" \
|
||||
"expected_internal_ip=$INSTANCE_INTERNAL_IP" \
|
||||
"active_state=$active_state" \
|
||||
"sub_state=$sub_state" \
|
||||
"nrestarts=$nrestarts" \
|
||||
"dispatcher_version=$DISPATCHER_VERSION" \
|
||||
"dispatcher_sha256=$dispatcher_hash"
|
||||
}
|
||||
|
||||
require_root() {
|
||||
[[ "$EUID" -eq 0 ]] || die "clone operations require the isolated OS Admin identity"
|
||||
[[ -n "${SUDO_USER:-}" && "$SUDO_USER" != "root" ]] || die "clone operation lacks a distinct OS Login caller"
|
||||
}
|
||||
|
||||
parity_receipt_relative() {
|
||||
local target_db="$1"
|
||||
printf 'docs/reports/leo-working-state-20260709/%s-canonical-parity-receipt.json\n' "$target_db"
|
||||
}
|
||||
|
||||
write_run_marker() {
|
||||
local marker="$1"
|
||||
local request_id="$2"
|
||||
local target_db="$3"
|
||||
printf 'request_id=%s\ntarget_db=%s\n' "$request_id" "$target_db" >"$marker"
|
||||
chmod 0600 "$marker"
|
||||
}
|
||||
|
||||
validate_run_dir() {
|
||||
local run_dir="$1"
|
||||
local request_id="$2"
|
||||
local target_db="$3"
|
||||
local marker="$run_dir/.run-owner"
|
||||
local resolved expected
|
||||
|
||||
[[ -d "$run_dir" && ! -L "$run_dir" ]] || die "exact run-owned directory is absent or unsafe"
|
||||
resolved="$(realpath "$run_dir")"
|
||||
expected="$REMOTE_RUN_ROOT/$request_id"
|
||||
[[ "$resolved" == "$expected" ]] || die "run-owned directory escaped its fixed root"
|
||||
[[ "$(stat -c '%U:%G:%a' "$run_dir")" == "root:root:700" ]] || die "run directory ownership or mode drifted"
|
||||
[[ -f "$marker" && ! -L "$marker" ]] || die "run ownership marker is absent or unsafe"
|
||||
[[ "$(cat "$marker")" == $'request_id='"$request_id"$'\ntarget_db='"$target_db" ]] || {
|
||||
die "run ownership marker does not match request_id and target_db"
|
||||
}
|
||||
}
|
||||
|
||||
remote_upload_path() {
|
||||
local request_id="$1"
|
||||
local passwd_entry home resolved upload
|
||||
|
||||
passwd_entry="$(getent passwd "$SUDO_USER")"
|
||||
[[ -n "$passwd_entry" ]] || die "OS Login caller has no passwd entry"
|
||||
home="$(cut -d: -f6 <<<"$passwd_entry")"
|
||||
[[ "$home" == /* && -d "$home" && ! -L "$home" ]] || die "OS Login caller home is unsafe"
|
||||
resolved="$(realpath "$home")"
|
||||
[[ "$resolved" == "$home" ]] || die "OS Login caller home path drifted"
|
||||
upload="$home/.teleo-iap-upload-$request_id.tar.gz"
|
||||
[[ -f "$upload" && ! -L "$upload" ]] || die "request-derived bundle upload is absent or unsafe"
|
||||
[[ "$(stat -c '%U:%a' "$upload")" == "$SUDO_USER:600" ]] || die "bundle upload owner or mode is unsafe"
|
||||
printf '%s\n' "$upload"
|
||||
}
|
||||
|
||||
validate_and_extract_bundle() {
|
||||
local archive="$1"
|
||||
local bundle_dir="$2"
|
||||
local operation="$3"
|
||||
local request_id="$4"
|
||||
local target_db="$5"
|
||||
|
||||
python3 - \
|
||||
"$archive" "$bundle_dir" "$operation" "$request_id" "$target_db" \
|
||||
"$REMOTE_DISPATCHER" "$DISPATCHER_VERSION" "$EXPECTED_WORKFLOW_REF" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
archive_path = Path(sys.argv[1])
|
||||
bundle_dir = Path(sys.argv[2]).resolve()
|
||||
operation, request_id, target_db = sys.argv[3:6]
|
||||
dispatcher_path = Path(sys.argv[6])
|
||||
dispatcher_version = sys.argv[7]
|
||||
expected_workflow_ref = sys.argv[8]
|
||||
common = ["scripts/gcp_iap_operator.sh"]
|
||||
direct_claim = [
|
||||
"scripts/run_gcp_generated_db_direct_claim_suite.py",
|
||||
"scripts/run_leo_clone_bound_handler_checkpoint.py",
|
||||
"scripts/working_leo_open_ended_benchmark.py",
|
||||
"hermes-agent/leoclean-bin/cloudsql_memory_tool.py",
|
||||
"ops/postgres_parity_manifest.sql",
|
||||
f"docs/reports/leo-working-state-20260709/{target_db}-canonical-parity-receipt.json",
|
||||
]
|
||||
expected = common + (direct_claim if operation == "direct-claim-replay" else [])
|
||||
if operation not in {"direct-claim-replay", "cleanup-clone"}:
|
||||
raise SystemExit("unexpected bundle operation")
|
||||
if archive_path.stat().st_size > 5 * 1024 * 1024:
|
||||
raise SystemExit("bundle archive exceeds size ceiling")
|
||||
|
||||
with tarfile.open(archive_path, mode="r:gz") as archive:
|
||||
members = archive.getmembers()
|
||||
expected_members = sorted([*expected, "bundle-manifest.json"])
|
||||
names = [member.name for member in members]
|
||||
if sorted(names) != expected_members or len(names) != len(set(names)):
|
||||
raise SystemExit("bundle member allowlist mismatch")
|
||||
if any(not member.isfile() or member.size > 2 * 1024 * 1024 for member in members):
|
||||
raise SystemExit("bundle contains a link, special entry, or oversized file")
|
||||
manifest_member = archive.getmember("bundle-manifest.json")
|
||||
if manifest_member.size > 64 * 1024:
|
||||
raise SystemExit("bundle manifest exceeds size ceiling")
|
||||
manifest = json.loads(archive.extractfile(manifest_member).read())
|
||||
required = {
|
||||
"schema": "livingip.gcpIapOperatorBundle.v1",
|
||||
"dispatcher_version": dispatcher_version,
|
||||
"operation": operation,
|
||||
"request_id": request_id,
|
||||
"target_db": target_db,
|
||||
"ref": "refs/heads/main",
|
||||
"workflow_ref": expected_workflow_ref,
|
||||
}
|
||||
if any(manifest.get(key) != value for key, value in required.items()):
|
||||
raise SystemExit("bundle manifest context mismatch")
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", str(manifest.get("git_commit") or "")):
|
||||
raise SystemExit("bundle manifest commit is invalid")
|
||||
files = manifest.get("files")
|
||||
if not isinstance(files, dict) or sorted(files) != sorted(expected):
|
||||
raise SystemExit("bundle manifest file allowlist mismatch")
|
||||
payloads = {}
|
||||
for relative in expected:
|
||||
member = archive.getmember(relative)
|
||||
data = archive.extractfile(member).read()
|
||||
receipt = files.get(relative) or {}
|
||||
if receipt.get("size") != len(data) or receipt.get("sha256") != hashlib.sha256(data).hexdigest():
|
||||
raise SystemExit(f"bundle hash mismatch: {relative}")
|
||||
payloads[relative] = data
|
||||
dispatcher_sha = hashlib.sha256(dispatcher_path.read_bytes()).hexdigest()
|
||||
if dispatcher_sha != files["scripts/gcp_iap_operator.sh"]["sha256"]:
|
||||
raise SystemExit("installed dispatcher does not match the reviewed main bundle")
|
||||
manifest_bytes = archive.extractfile(manifest_member).read()
|
||||
|
||||
for relative, data in sorted({**payloads, "bundle-manifest.json": manifest_bytes}.items()):
|
||||
destination = (bundle_dir / relative).resolve()
|
||||
if bundle_dir not in destination.parents:
|
||||
raise SystemExit("bundle destination escaped the run directory")
|
||||
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
os.chmod(destination.parent, 0o700)
|
||||
with destination.open("xb") as handle:
|
||||
handle.write(data)
|
||||
os.chmod(destination, 0o600)
|
||||
PY
|
||||
}
|
||||
|
||||
receive_and_validate_bundle() {
|
||||
local operation="$1"
|
||||
local request_id="$2"
|
||||
local target_db="$3"
|
||||
local upload run_dir archive bundle_dir
|
||||
|
||||
require_root
|
||||
upload="$(remote_upload_path "$request_id")"
|
||||
run_dir="$REMOTE_RUN_ROOT/$request_id"
|
||||
if [[ "$operation" == "direct-claim-replay" ]]; then
|
||||
[[ ! -e "$run_dir" ]] || die "request_id already owns a remote run directory"
|
||||
install -d -o root -g root -m 0700 "$run_dir"
|
||||
write_run_marker "$run_dir/.run-owner" "$request_id" "$target_db"
|
||||
else
|
||||
validate_run_dir "$run_dir" "$request_id" "$target_db"
|
||||
fi
|
||||
archive="$run_dir/$operation-bundle.tar.gz"
|
||||
bundle_dir="$run_dir/$operation-bundle"
|
||||
[[ ! -e "$archive" && ! -e "$bundle_dir" ]] || die "operation bundle already exists in run directory"
|
||||
mv -- "$upload" "$archive"
|
||||
chown root:root "$archive"
|
||||
chmod 0600 "$archive"
|
||||
install -d -o root -g root -m 0700 "$bundle_dir"
|
||||
validate_and_extract_bundle "$archive" "$bundle_dir" "$operation" "$request_id" "$target_db" || {
|
||||
die "bundle membership, hash, context, or dispatcher validation failed"
|
||||
}
|
||||
VALIDATED_BUNDLE_DIR="$bundle_dir"
|
||||
VALIDATED_BUNDLE_SHA256="$(sha256sum "$archive" | awk '{print $1}')"
|
||||
VALIDATED_BUNDLE_COMMIT="$(
|
||||
python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["git_commit"])' \
|
||||
"$bundle_dir/bundle-manifest.json"
|
||||
)"
|
||||
}
|
||||
|
||||
remote_direct_claim_replay() {
|
||||
local request_id="$1"
|
||||
local target_db="$2"
|
||||
local receipt_relative receipt_path run_dir output_path stdout_path stderr_path report_sha result_bytes
|
||||
|
||||
receive_and_validate_bundle "direct-claim-replay" "$request_id" "$target_db"
|
||||
receipt_relative="$(parity_receipt_relative "$target_db")"
|
||||
receipt_path="$VALIDATED_BUNDLE_DIR/$receipt_relative"
|
||||
[[ -s "$receipt_path" && ! -L "$receipt_path" ]] || die "target-specific tracked parity receipt is absent"
|
||||
[[ -d "$LIVE_PROFILE" ]] || die "live profile required for temporary copy is absent"
|
||||
|
||||
run_dir="$REMOTE_RUN_ROOT/$request_id"
|
||||
output_path="$run_dir/direct-claim-result.json"
|
||||
stdout_path="$run_dir/direct-claim.stdout"
|
||||
stderr_path="$run_dir/direct-claim.stderr"
|
||||
: >"$stdout_path"
|
||||
: >"$stderr_path"
|
||||
chmod 0600 "$stdout_path" "$stderr_path"
|
||||
|
||||
if ! (
|
||||
cd "$VALIDATED_BUNDLE_DIR"
|
||||
/usr/bin/python3 scripts/run_gcp_generated_db_direct_claim_suite.py \
|
||||
--execute \
|
||||
--target-db "$target_db" \
|
||||
--cloudsql-tool hermes-agent/leoclean-bin/cloudsql_memory_tool.py \
|
||||
--manifest-sql ops/postgres_parity_manifest.sql \
|
||||
--parity-receipt "$receipt_path" \
|
||||
--output "$output_path" \
|
||||
--host "$CLOUDSQL_HOST" \
|
||||
--project "$PROJECT_ID" \
|
||||
--password-secret "$CLOUDSQL_SECRET" \
|
||||
--service "$LIVE_SERVICE" \
|
||||
--live-profile "$LIVE_PROFILE" \
|
||||
--run-id "$request_id"
|
||||
) >"$stdout_path" 2>"$stderr_path"; then
|
||||
die "reviewed six-turn clone-bound replay failed; private run files were retained for cleanup"
|
||||
fi
|
||||
|
||||
[[ -s "$output_path" && ! -L "$output_path" ]] || die "six-turn replay did not retain its private receipt"
|
||||
python3 - "$output_path" "$target_db" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
payload = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
|
||||
target_db = sys.argv[2]
|
||||
checks = payload.get("checks") or {}
|
||||
required = {
|
||||
"status": payload.get("status") == "pass",
|
||||
"target": payload.get("target_database") == target_db,
|
||||
"six_replies": checks.get("six_replies_returned") is True,
|
||||
"no_message_send": payload.get("posted_to_telegram") is False,
|
||||
"service_unchanged": checks.get("live_service_unchanged") is True,
|
||||
"profile_unchanged": checks.get("live_profile_unchanged") is True,
|
||||
"database_unchanged": checks.get("generated_database_unchanged") is True,
|
||||
"temporary_profile_removed": checks.get("temporary_profile_removed") is True,
|
||||
}
|
||||
failed = sorted(name for name, ok in required.items() if not ok)
|
||||
if failed:
|
||||
raise SystemExit("replay receipt failed fixed checks: " + ", ".join(failed))
|
||||
PY
|
||||
chmod 0600 "$output_path"
|
||||
report_sha="$(sha256sum "$output_path" | awk '{print $1}')"
|
||||
result_bytes="$(wc -c <"$output_path" | tr -d ' ')"
|
||||
emit_json \
|
||||
"direct-claim-replay" "$request_id" "$target_db" "pass" \
|
||||
"bundle_commit=$VALIDATED_BUNDLE_COMMIT" \
|
||||
"bundle_sha256=$VALIDATED_BUNDLE_SHA256" \
|
||||
"result_sha256=$report_sha" \
|
||||
"result_bytes=$result_bytes" \
|
||||
"no_message_send=true" \
|
||||
"live_service_unchanged=true" \
|
||||
"live_profile_unchanged=true"
|
||||
}
|
||||
|
||||
remote_cleanup_clone() {
|
||||
local request_id="$1"
|
||||
local target_db="$2"
|
||||
local run_dir password exists remaining
|
||||
|
||||
receive_and_validate_bundle "cleanup-clone" "$request_id" "$target_db"
|
||||
run_dir="$REMOTE_RUN_ROOT/$request_id"
|
||||
validate_run_dir "$run_dir" "$request_id" "$target_db"
|
||||
command -v gcloud >/dev/null 2>&1 || die "gcloud is unavailable for secret-backed clone cleanup"
|
||||
command -v psql >/dev/null 2>&1 || die "psql is unavailable for clone cleanup"
|
||||
|
||||
password="$(gcloud secrets versions access latest --secret="$CLOUDSQL_SECRET" --project="$PROJECT_ID" --quiet)"
|
||||
[[ -n "$password" ]] || die "clone cleanup credential resolved empty"
|
||||
export PGPASSWORD="$password"
|
||||
export PGOPTIONS="-c statement_timeout=30000"
|
||||
exists="$(
|
||||
psql "host=$CLOUDSQL_HOST port=5432 dbname=postgres user=postgres sslmode=require connect_timeout=8" \
|
||||
-X -Atq -v ON_ERROR_STOP=1 -v "target_db=$target_db" \
|
||||
-c "select exists(select 1 from pg_database where datname = :'target_db');"
|
||||
)"
|
||||
[[ "$exists" == "t" ]] || die "marked clone database is absent; refusing directory-only cleanup"
|
||||
|
||||
psql "host=$CLOUDSQL_HOST port=5432 dbname=postgres user=postgres sslmode=require connect_timeout=8" \
|
||||
-X -q -v ON_ERROR_STOP=1 -v "target_db=$target_db" <<'SQL'
|
||||
select pg_terminate_backend(pid)
|
||||
from pg_stat_activity
|
||||
where datname = :'target_db' and pid <> pg_backend_pid();
|
||||
drop database :"target_db";
|
||||
SQL
|
||||
remaining="$(
|
||||
psql "host=$CLOUDSQL_HOST port=5432 dbname=postgres user=postgres sslmode=require connect_timeout=8" \
|
||||
-X -Atq -v ON_ERROR_STOP=1 -v "target_db=$target_db" \
|
||||
-c "select count(*) from pg_database where datname = :'target_db';"
|
||||
)"
|
||||
unset PGPASSWORD PGOPTIONS
|
||||
password=""
|
||||
[[ "$remaining" == "0" ]] || die "clone database remained after cleanup"
|
||||
|
||||
validate_run_dir "$run_dir" "$request_id" "$target_db"
|
||||
rm -rf --one-file-system -- "$run_dir"
|
||||
[[ ! -e "$run_dir" ]] || die "exact run-owned directory remained after cleanup"
|
||||
emit_json \
|
||||
"cleanup-clone" "$request_id" "$target_db" "pass" \
|
||||
"bundle_commit=$VALIDATED_BUNDLE_COMMIT" \
|
||||
"bundle_sha256=$VALIDATED_BUNDLE_SHA256" \
|
||||
"clone_database_remaining=0"
|
||||
}
|
||||
|
||||
remote_main() {
|
||||
[[ "$#" -eq 3 ]] || die "remote mode requires operation, request_id, and target_db"
|
||||
local operation="$1"
|
||||
local request_id="$2"
|
||||
local target_db="$3"
|
||||
validate_inputs "$operation" "$request_id" "$target_db"
|
||||
case "$operation" in
|
||||
status) remote_status "$request_id" "$target_db" ;;
|
||||
direct-claim-replay) remote_direct_claim_replay "$request_id" "$target_db" ;;
|
||||
cleanup-clone) remote_cleanup_clone "$request_id" "$target_db" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
write_sanitized_result() {
|
||||
local output_path="$1"
|
||||
local raw_stdout="$2"
|
||||
local operation="$3"
|
||||
local request_id="$4"
|
||||
local target_db="$5"
|
||||
local exit_code="$6"
|
||||
RESULT_PATH="$output_path" RAW_STDOUT="$raw_stdout" OPERATION="$operation" \
|
||||
REQUEST_ID="$request_id" TARGET_DB="$target_db" EXIT_CODE="$exit_code" python3 - <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
raw = Path(os.environ["RAW_STDOUT"]).read_bytes()
|
||||
exit_code = int(os.environ["EXIT_CODE"])
|
||||
remote = {}
|
||||
if exit_code == 0:
|
||||
try:
|
||||
candidate = json.loads(raw.decode("utf-8").splitlines()[-1])
|
||||
allowed = {
|
||||
"operation",
|
||||
"request_id",
|
||||
"target_db",
|
||||
"status",
|
||||
"instance",
|
||||
"expected_internal_ip",
|
||||
"active_state",
|
||||
"sub_state",
|
||||
"nrestarts",
|
||||
"dispatcher_version",
|
||||
"dispatcher_sha256",
|
||||
"bundle_commit",
|
||||
"bundle_sha256",
|
||||
"result_sha256",
|
||||
"result_bytes",
|
||||
"no_message_send",
|
||||
"live_service_unchanged",
|
||||
"live_profile_unchanged",
|
||||
"clone_database_remaining",
|
||||
}
|
||||
remote = {key: value for key, value in candidate.items() if key in allowed}
|
||||
except (UnicodeDecodeError, json.JSONDecodeError, IndexError):
|
||||
exit_code = 90
|
||||
|
||||
payload = {
|
||||
"schema": "livingip.gcpIapOperatorSanitizedResult.v1",
|
||||
"operation": os.environ["OPERATION"],
|
||||
"request_id": os.environ["REQUEST_ID"],
|
||||
"target_db": os.environ["TARGET_DB"],
|
||||
"status": "pass" if exit_code == 0 and remote.get("status") == "pass" else "fail",
|
||||
"ssh_exit_code": exit_code,
|
||||
"remote_result": remote,
|
||||
"remote_stdout_sha256": hashlib.sha256(raw).hexdigest(),
|
||||
"remote_stdout_bytes": len(raw),
|
||||
"raw_stdout_retained": False,
|
||||
"raw_stderr_retained": False,
|
||||
"credential_values_logged": False,
|
||||
"ssh_debug_enabled": False,
|
||||
}
|
||||
Path(os.environ["RESULT_PATH"]).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
PY
|
||||
chmod 0600 "$output_path"
|
||||
}
|
||||
|
||||
validate_local_bundle() {
|
||||
local request_id="$1"
|
||||
local expected path resolved bundle_root mode
|
||||
|
||||
bundle_root="$RUNNER_TEMP/gcp-iap-operator-bundle"
|
||||
expected="$bundle_root/$request_id.tar.gz"
|
||||
path="$expected"
|
||||
[[ -f "$path" && ! -L "$path" ]] || die "fixed workflow bundle is absent or unsafe"
|
||||
resolved="$(realpath "$path")"
|
||||
[[ "$resolved" == "$expected" ]] || die "workflow bundle escaped its request-derived path"
|
||||
mode="$(python3 -c 'import os,stat,sys; print(oct(stat.S_IMODE(os.stat(sys.argv[1]).st_mode))[2:])' "$path")"
|
||||
[[ "$mode" == "600" ]] || die "workflow bundle mode is not private"
|
||||
printf '%s\n' "$path"
|
||||
}
|
||||
|
||||
runner_main() {
|
||||
[[ "$#" -eq 3 ]] || die "runner mode requires operation, request_id, and target_db"
|
||||
local operation="$1"
|
||||
local request_id="$2"
|
||||
local target_db="$3"
|
||||
local temp_dir result_dir result_path raw_stdout raw_stderr scp_stdout ssh_key remote_command exit_code
|
||||
local sanitized_status bundle_path remote_upload
|
||||
local -a remote_argv
|
||||
|
||||
validate_inputs "$operation" "$request_id" "$target_db"
|
||||
[[ -n "${RUNNER_TEMP:-}" && "$RUNNER_TEMP" == /* ]] || die "RUNNER_TEMP must be an absolute runner path"
|
||||
[[ -n "${RESULT_DIR:-}" && "$RESULT_DIR" == "$RUNNER_TEMP"/* ]] || {
|
||||
die "RESULT_DIR must be a fixed child of RUNNER_TEMP"
|
||||
}
|
||||
command -v gcloud >/dev/null 2>&1 || die "gcloud is unavailable"
|
||||
install -d -m 0700 "$RESULT_DIR"
|
||||
result_dir="$(realpath "$RESULT_DIR")"
|
||||
[[ "$result_dir" == "$RUNNER_TEMP"/* ]] || die "RESULT_DIR escaped RUNNER_TEMP"
|
||||
result_path="$result_dir/result.json"
|
||||
: >"$result_path"
|
||||
chmod 0600 "$result_path"
|
||||
|
||||
temp_dir="$(mktemp -d "$RUNNER_TEMP/teleo-iap-operator.XXXXXX")"
|
||||
RUNNER_TEMP_DIR="$temp_dir"
|
||||
chmod 0700 "$temp_dir"
|
||||
trap cleanup_runner_temp EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
raw_stdout="$temp_dir/remote.stdout"
|
||||
raw_stderr="$temp_dir/transport.stderr"
|
||||
scp_stdout="$temp_dir/scp.stdout"
|
||||
ssh_key="$temp_dir/ssh_key"
|
||||
: >"$raw_stdout"
|
||||
: >"$raw_stderr"
|
||||
: >"$scp_stdout"
|
||||
chmod 0600 "$raw_stdout" "$raw_stderr" "$scp_stdout"
|
||||
|
||||
if [[ "$operation" != "status" ]]; then
|
||||
bundle_path="$(validate_local_bundle "$request_id")"
|
||||
remote_upload="$INSTANCE:.teleo-iap-upload-$request_id.tar.gz"
|
||||
set +e
|
||||
gcloud compute scp "$bundle_path" "$remote_upload" \
|
||||
--project="$PROJECT_ID" \
|
||||
--zone="$ZONE" \
|
||||
--tunnel-through-iap \
|
||||
--ssh-key-file="$ssh_key" \
|
||||
--ssh-key-expire-after=5m \
|
||||
--scp-flag=-p \
|
||||
--quiet >"$scp_stdout" 2>>"$raw_stderr"
|
||||
exit_code=$?
|
||||
set -e
|
||||
if [[ "$exit_code" -ne 0 ]]; then
|
||||
write_sanitized_result "$result_path" "$raw_stdout" "$operation" "$request_id" "$target_db" "$exit_code"
|
||||
printf 'sanitized_result=%s\n' "$result_path"
|
||||
return "$exit_code"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$operation" == "status" ]]; then
|
||||
remote_argv=("$REMOTE_DISPATCHER" --remote "$operation" "$request_id" "$target_db")
|
||||
else
|
||||
remote_argv=(sudo -n "$REMOTE_DISPATCHER" --remote "$operation" "$request_id" "$target_db")
|
||||
fi
|
||||
printf -v remote_command '%q ' "${remote_argv[@]}"
|
||||
|
||||
set +e
|
||||
gcloud compute ssh "$INSTANCE" \
|
||||
--project="$PROJECT_ID" \
|
||||
--zone="$ZONE" \
|
||||
--tunnel-through-iap \
|
||||
--ssh-key-file="$ssh_key" \
|
||||
--ssh-key-expire-after=5m \
|
||||
--command="$remote_command" \
|
||||
--quiet >"$raw_stdout" 2>>"$raw_stderr"
|
||||
exit_code=$?
|
||||
set -e
|
||||
[[ ! -e "$ssh_key" ]] || chmod 0600 "$ssh_key"
|
||||
[[ ! -e "$ssh_key.pub" ]] || chmod 0600 "$ssh_key.pub"
|
||||
write_sanitized_result "$result_path" "$raw_stdout" "$operation" "$request_id" "$target_db" "$exit_code"
|
||||
sanitized_status="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["status"])' "$result_path")"
|
||||
if [[ "$exit_code" -eq 0 && "$sanitized_status" != "pass" ]]; then
|
||||
exit_code=90
|
||||
fi
|
||||
printf 'sanitized_result=%s\n' "$result_path"
|
||||
return "$exit_code"
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "--remote" ]]; then
|
||||
shift
|
||||
remote_main "$@"
|
||||
else
|
||||
runner_main "$@"
|
||||
fi
|
||||
1772
scripts/run_gcp_generated_db_working_leo_suite.py
Normal file
1772
scripts/run_gcp_generated_db_working_leo_suite.py
Normal file
File diff suppressed because it is too large
Load diff
645
tests/test_gcp_generated_db_working_leo_suite.py
Normal file
645
tests/test_gcp_generated_db_working_leo_suite.py
Normal file
|
|
@ -0,0 +1,645 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts import run_gcp_generated_db_working_leo_suite as suite
|
||||
|
||||
|
||||
def parity_payload(target_db: str = "teleo_clone_working_leo") -> dict[str, object]:
|
||||
return {
|
||||
"artifact": "canonical_postgres_parity_verification",
|
||||
"status": "pass",
|
||||
"scope": "disposable_cloudsql_copy",
|
||||
"problems": [],
|
||||
"details": {
|
||||
"target_database": target_db,
|
||||
"source_table_count": 39,
|
||||
"target_table_count": 39,
|
||||
"source_total_rows": 100,
|
||||
"target_total_rows": 100,
|
||||
"table_mismatches": [],
|
||||
"application_role_mismatches": {},
|
||||
"required_extension_mismatches": {},
|
||||
"performance_problems": [],
|
||||
"structural_hashes": {
|
||||
"columns": {"source": "same", "target": "same"},
|
||||
"constraints": {"source": "same", "target": "same"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_parity(tmp_path: Path, target_db: str = "teleo_clone_working_leo") -> Path:
|
||||
path = tmp_path / "parity.json"
|
||||
path.write_text(json.dumps(parity_payload(target_db)), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def write_pinned_json(tmp_path: Path, name: str, payload: dict[str, object]) -> tuple[Path, str]:
|
||||
path = tmp_path / name
|
||||
content = json.dumps(payload, sort_keys=True).encode()
|
||||
path.write_bytes(content)
|
||||
return path, hashlib.sha256(content).hexdigest()
|
||||
|
||||
|
||||
def target_identity_payload(**overrides: object) -> dict[str, object]:
|
||||
payload: dict[str, object] = {
|
||||
"artifact": suite.TARGET_IDENTITY_ARTIFACT,
|
||||
"status": "pass",
|
||||
"problems": [],
|
||||
"target_database": "teleo_clone_working_leo",
|
||||
"project": "teleo-501523",
|
||||
"instance": "teleo-pgvector-standby",
|
||||
"server_address": "10.61.0.3",
|
||||
"system_identifier": "target-system-123",
|
||||
"captured_at_utc": "2026-07-12T00:00:00Z",
|
||||
}
|
||||
payload.update(overrides)
|
||||
return payload
|
||||
|
||||
|
||||
def source_baseline_payload() -> dict[str, object]:
|
||||
tables = suite.composition.PRODUCTION_TABLES
|
||||
return {
|
||||
"artifact": suite.SOURCE_BASELINE_ARTIFACT,
|
||||
"status": "pass",
|
||||
"problems": [],
|
||||
"project": "teleo-501523",
|
||||
"instance": "teleo-pgvector-standby",
|
||||
"captured_at_utc": "2026-07-12T00:00:00Z",
|
||||
"guard_snapshot": {
|
||||
"database_identity": {
|
||||
"current_database": "teleo_source",
|
||||
"system_identifier": "source-system-456",
|
||||
},
|
||||
"counts": {table: 1 for table in tables},
|
||||
"rowset_md5": {table: f"md5-{index}" for index, table in enumerate(tables)},
|
||||
},
|
||||
"gate_schema": {
|
||||
"approval_table": "kb_stage.kb_proposal_approvals",
|
||||
"review_principals_table": "kb_stage.kb_review_principals",
|
||||
"review_role_exists": True,
|
||||
"apply_role_exists": True,
|
||||
"approve_function": "approve",
|
||||
"assert_function": "assert",
|
||||
"finish_function": "finish",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def capability_receipt(
|
||||
role: str, target_db: str = "teleo_clone_working_leo", *, writable: bool | None = None
|
||||
) -> str:
|
||||
capabilities = {
|
||||
"kb_read": (False, False, False, False, False),
|
||||
"kb_stage_writer": (True, False, False, False, False),
|
||||
"kb_review": (False, True, False, False, False),
|
||||
"kb_apply": (False, False, True, True, True),
|
||||
}
|
||||
insert_proposals, approve, assert_apply, finish_apply, insert_canonical = capabilities[role]
|
||||
writable = role != "kb_read" if writable is None else writable
|
||||
payload = {
|
||||
"session_user": role,
|
||||
"current_user": role,
|
||||
"current_database": target_db,
|
||||
"transaction_read_only": "off" if writable else "on",
|
||||
"default_transaction_read_only": "off" if writable else "on",
|
||||
"is_superuser": False,
|
||||
"can_select_proposals": True,
|
||||
"can_insert_proposals": insert_proposals,
|
||||
"can_update_proposals": False,
|
||||
"can_approve": approve,
|
||||
"can_assert_apply": assert_apply,
|
||||
"can_finish_apply": finish_apply,
|
||||
"can_insert_canonical": insert_canonical,
|
||||
}
|
||||
return suite.ROLE_RECEIPT_PREFIX + json.dumps(payload, sort_keys=True)
|
||||
|
||||
|
||||
def executor_args(**overrides: object) -> argparse.Namespace:
|
||||
values: dict[str, object] = {
|
||||
"target_db": "teleo_clone_working_leo",
|
||||
"host": "10.61.0.3",
|
||||
"project": "teleo-501523",
|
||||
"instance": "teleo-pgvector-standby",
|
||||
"password_secret": "read-secret",
|
||||
"read_role": "kb_read",
|
||||
"stage_role": "kb_stage_writer",
|
||||
"review_role": "kb_review",
|
||||
"apply_role": "kb_apply",
|
||||
"stage_password_secret": "stage-secret",
|
||||
"review_password_secret": "review-secret",
|
||||
"apply_password_secret": "apply-secret",
|
||||
"allow_stage": True,
|
||||
"allow_review": True,
|
||||
"allow_apply": True,
|
||||
}
|
||||
values.update(overrides)
|
||||
return argparse.Namespace(**values)
|
||||
|
||||
|
||||
def test_reuses_exact_twenty_prompt_catalog_and_existing_scorer_contracts() -> None:
|
||||
catalog = suite.full_catalog()
|
||||
checks = suite.composition_check_contract()
|
||||
|
||||
assert len(catalog) == 20
|
||||
assert [row["id"] for row in catalog] == [
|
||||
"OE-01",
|
||||
"OE-02",
|
||||
"OE-03",
|
||||
"OE-04",
|
||||
"OE-05",
|
||||
"CS-01",
|
||||
"CS-02",
|
||||
"CS-03",
|
||||
"CS-04",
|
||||
"CS-05",
|
||||
"CS-06",
|
||||
"CS-07",
|
||||
"CS-08",
|
||||
"CS-09",
|
||||
"DC-01",
|
||||
"DC-02",
|
||||
"DC-03",
|
||||
"DC-04",
|
||||
"DC-05",
|
||||
"DC-06",
|
||||
]
|
||||
assert len(checks) == 34
|
||||
assert "source_packet_validated" in checks
|
||||
assert "isolated_restart_and_memory_survived" in checks
|
||||
|
||||
|
||||
def test_validation_binds_parity_and_reviewed_hashes_to_generated_database(tmp_path: Path) -> None:
|
||||
args = argparse.Namespace(
|
||||
target_db="teleo_clone_working_leo",
|
||||
cloudsql_tool=Path("hermes-agent/leoclean-bin/cloudsql_memory_tool.py"),
|
||||
manifest_sql=Path("ops/postgres_parity_manifest.sql"),
|
||||
parity_receipt=write_parity(tmp_path),
|
||||
)
|
||||
|
||||
result = suite.validate_local_inputs(args)
|
||||
|
||||
assert result["parity_receipt"]["target_database"] == args.target_db
|
||||
assert result["catalog_contract"]["count"] == 20
|
||||
assert result["composition_contract"]["inherited_check_count"] == 34
|
||||
assert result["composition_contract"]["inherited_34_of_34_claimable"] is False
|
||||
assert result["composition_contract"]["gcp_required_check_count"] == 32
|
||||
assert result["execute_input_contract"]["ready"] is False
|
||||
assert "--target-identity-receipt" in result["execute_input_contract"]["missing"]
|
||||
assert result["reviewed_inputs"]["cloudsql_tool_sha256"] == suite.gcp.REVIEWED_CLOUDSQL_TOOL_SHA256
|
||||
assert result["reviewed_inputs"]["manifest_sql_sha256"] == suite.gcp.REVIEWED_MANIFEST_SQL_SHA256
|
||||
|
||||
args.target_db = "teleo_clone_other"
|
||||
with pytest.raises(RuntimeError, match="target_database"):
|
||||
suite.validate_local_inputs(args)
|
||||
|
||||
|
||||
def test_executor_binds_every_call_and_defaults_read_only_except_controller_phases(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls: list[dict[str, object]] = []
|
||||
|
||||
def fake_run(command: list[str], **kwargs: object) -> str:
|
||||
if command[0] == "gcloud":
|
||||
return "private-value\n"
|
||||
if "env" in kwargs:
|
||||
kwargs["env"] = dict(kwargs["env"])
|
||||
calls.append({"command": command, **kwargs})
|
||||
connection = command[1]
|
||||
role = connection.split("user=", 1)[1].split(" ", 1)[0]
|
||||
writable = kwargs["env"]["PGOPTIONS"].endswith("off")
|
||||
return capability_receipt(role, writable=writable) + "\nok\n"
|
||||
|
||||
monkeypatch.setattr(suite.gcp, "run", fake_run)
|
||||
executor = suite.CloudSqlExecutor(executor_args())
|
||||
|
||||
executor.read("select 1;")
|
||||
executor.stage("insert into kb_stage.kb_proposals default values;")
|
||||
executor.review("select kb_stage.approve_strict_proposal();")
|
||||
executor.apply("select kb_stage.finish_approved_proposal();")
|
||||
|
||||
assert len(calls) == 4
|
||||
assert all("dbname=teleo_clone_working_leo" in call["command"][1] for call in calls)
|
||||
assert all("expected_database=teleo_clone_working_leo" in call["command"] for call in calls)
|
||||
assert calls[0]["env"]["PGOPTIONS"] == "-c default_transaction_read_only=on"
|
||||
assert [call["env"]["PGOPTIONS"] for call in calls[1:]] == [
|
||||
"-c default_transaction_read_only=off",
|
||||
"-c default_transaction_read_only=off",
|
||||
"-c default_transaction_read_only=off",
|
||||
]
|
||||
assert all("refusing database fallback" in str(call["input_text"]) for call in calls)
|
||||
assert [receipt["purpose"] for receipt in executor.receipts] == ["read", "stage", "review", "apply"]
|
||||
assert [receipt["capability_receipt"]["session_user"] for receipt in executor.receipts] == [
|
||||
"kb_read",
|
||||
"kb_stage_writer",
|
||||
"kb_review",
|
||||
"kb_apply",
|
||||
]
|
||||
|
||||
|
||||
def test_executor_refuses_unflagged_or_mislabeled_write_authority(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(suite.gcp, "run", lambda *_args, **_kwargs: "secret\n")
|
||||
executor = suite.CloudSqlExecutor(executor_args(allow_apply=False))
|
||||
|
||||
with pytest.raises(RuntimeError, match="not explicitly authorized"):
|
||||
executor.apply("select 1;")
|
||||
with pytest.raises(RuntimeError, match="write mode is allowed only"):
|
||||
executor.execute(
|
||||
"select 1;",
|
||||
role="postgres",
|
||||
password_secret="read-secret",
|
||||
purpose="read",
|
||||
writable=True,
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="safe identifier"):
|
||||
executor.execute(
|
||||
"select 1;",
|
||||
role="postgres dbname=teleo",
|
||||
password_secret="read-secret",
|
||||
)
|
||||
postgres_writer = suite.CloudSqlExecutor(executor_args(stage_role="postgres"))
|
||||
with pytest.raises(RuntimeError, match="postgres write authority"):
|
||||
postgres_writer.stage("select 1;")
|
||||
|
||||
|
||||
def test_sha_pinned_target_and_source_receipts_bind_physical_context(tmp_path: Path) -> None:
|
||||
target_path, target_sha = write_pinned_json(tmp_path, "target.json", target_identity_payload())
|
||||
source_path, source_sha = write_pinned_json(tmp_path, "source.json", source_baseline_payload())
|
||||
args = argparse.Namespace(
|
||||
validation_only=True,
|
||||
execute=False,
|
||||
target_db="teleo_clone_working_leo",
|
||||
host="10.61.0.3",
|
||||
project="teleo-501523",
|
||||
instance="teleo-pgvector-standby",
|
||||
cloudsql_tool=Path("hermes-agent/leoclean-bin/cloudsql_memory_tool.py"),
|
||||
manifest_sql=Path("ops/postgres_parity_manifest.sql"),
|
||||
parity_receipt=write_parity(tmp_path),
|
||||
target_identity_receipt=target_path,
|
||||
target_identity_sha256=target_sha,
|
||||
source_baseline_receipt=source_path,
|
||||
source_baseline_sha256=source_sha,
|
||||
)
|
||||
|
||||
result = suite.validate_local_inputs(args)
|
||||
|
||||
assert result["execute_input_contract"] == {"ready": True, "missing": []}
|
||||
assert result["target_identity_receipt"]["system_identifier"] == "target-system-123"
|
||||
assert result["source_baseline_receipt"]["source_system_identifier"] == "source-system-456"
|
||||
suite.assert_live_target_identity(
|
||||
args,
|
||||
result["target_identity_receipt"],
|
||||
{
|
||||
"current_database": args.target_db,
|
||||
"system_identifier": "target-system-123",
|
||||
"server_address": args.host,
|
||||
"ssl": True,
|
||||
"default_transaction_read_only": "on",
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="system_identifier"):
|
||||
suite.assert_live_target_identity(
|
||||
args,
|
||||
result["target_identity_receipt"],
|
||||
{
|
||||
"current_database": args.target_db,
|
||||
"system_identifier": "different-physical-cluster",
|
||||
"server_address": args.host,
|
||||
"ssl": True,
|
||||
"default_transaction_read_only": "on",
|
||||
},
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="SHA-256 pin"):
|
||||
suite.validate_target_identity_receipt(args, target_path, "0" * 64)
|
||||
|
||||
|
||||
def test_source_snapshots_are_receipt_backed_and_inherited_source_checks_are_not_claimed() -> None:
|
||||
args = executor_args()
|
||||
source_payload = source_baseline_payload()
|
||||
source = {
|
||||
"guard_snapshot": source_payload["guard_snapshot"],
|
||||
"gate_schema": source_payload["gate_schema"],
|
||||
}
|
||||
adapter = suite.RuntimeAdapter(args, suite.CloudSqlExecutor(args), source, {"system_identifier": "target"})
|
||||
|
||||
assert adapter._guard_snapshot(
|
||||
suite.bound.PRODUCTION_CONTAINER,
|
||||
suite.bound.PRODUCTION_DB,
|
||||
tables=suite.composition.PRODUCTION_TABLES,
|
||||
) == source_payload["guard_snapshot"]
|
||||
assert adapter._gate_schema(suite.bound.PRODUCTION_CONTAINER, suite.bound.PRODUCTION_DB) == source_payload[
|
||||
"gate_schema"
|
||||
]
|
||||
contract = suite.composition_check_contract()
|
||||
required = [name for name in contract if name not in suite.UNSUPPORTED_INHERITED_COMPOSITION_CHECKS]
|
||||
assert len(contract) == 34
|
||||
assert len(required) == 32
|
||||
assert "production_database_unchanged" not in required
|
||||
assert "production_gate_schema_unchanged" not in required
|
||||
|
||||
|
||||
def test_duplicate_credential_values_rejected_without_retaining_values(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
executor = suite.CloudSqlExecutor(executor_args())
|
||||
dummy = "DUMMY-DUPLICATE-CREDENTIAL"
|
||||
monkeypatch.setattr(executor, "_password", lambda _name: dummy)
|
||||
|
||||
with pytest.raises(RuntimeError, match="credential values must be distinct") as captured:
|
||||
suite.verify_distinct_credential_values(executor)
|
||||
|
||||
assert dummy not in str(captured.value)
|
||||
|
||||
|
||||
def test_catalog_top_level_gate_requires_status_and_every_safety_check() -> None:
|
||||
assert suite.catalog_gate_pass({"status": "pass", "checks": {"score": True, "no_send": True}}) is True
|
||||
assert suite.catalog_gate_pass({"status": "fail", "checks": {"score": True, "no_send": True}}) is False
|
||||
assert suite.catalog_gate_pass({"status": "pass", "checks": {"score": True, "no_send": False}}) is False
|
||||
assert suite.catalog_gate_pass({"status": "pass", "checks": {}}) is False
|
||||
|
||||
|
||||
def test_prompt_scoped_evidence_requires_matching_nonce_target_read_and_role(tmp_path: Path) -> None:
|
||||
path = tmp_path / "tool.jsonl"
|
||||
identity = {
|
||||
"current_database": "teleo_clone_working_leo",
|
||||
"system_identifier": "target-system-123",
|
||||
"transaction_read_only": "on",
|
||||
"default_transaction_read_only": "on",
|
||||
"session_user": "kb_read",
|
||||
"current_user": "kb_read",
|
||||
}
|
||||
events = [
|
||||
{
|
||||
"phase": "start",
|
||||
"prompt_id": "OE-01",
|
||||
"run_nonce": "nonce-1",
|
||||
"invocation_id": "invocation-1",
|
||||
"container": suite.gcp.SOURCE_COMPUTE,
|
||||
"database": "teleo_clone_working_leo",
|
||||
"database_identity": identity,
|
||||
"argv": ["search", "working leo"],
|
||||
},
|
||||
{
|
||||
"phase": "end",
|
||||
"prompt_id": "OE-01",
|
||||
"run_nonce": "nonce-1",
|
||||
"invocation_id": "invocation-1",
|
||||
"container": suite.gcp.SOURCE_COMPUTE,
|
||||
"database": "teleo_clone_working_leo",
|
||||
"returncode": 0,
|
||||
},
|
||||
]
|
||||
path.write_text("".join(json.dumps(event) + "\n" for event in events), encoding="utf-8")
|
||||
|
||||
proof = suite.prompt_scoped_tool_evidence(
|
||||
path,
|
||||
prompt_id="OE-01",
|
||||
run_nonce="nonce-1",
|
||||
target_db="teleo_clone_working_leo",
|
||||
db_identity=identity,
|
||||
read_role="kb_read",
|
||||
)
|
||||
|
||||
assert proof["pass"] is True
|
||||
mismatched = suite.prompt_scoped_tool_evidence(
|
||||
path,
|
||||
prompt_id="OE-02",
|
||||
run_nonce="nonce-1",
|
||||
target_db="teleo_clone_working_leo",
|
||||
db_identity=identity,
|
||||
read_role="kb_read",
|
||||
)
|
||||
assert mismatched["pass"] is False
|
||||
assert mismatched["checks"]["prompt_and_run_nonce_match"] is False
|
||||
|
||||
|
||||
def test_dc_sessions_must_use_unique_fresh_profiles_and_sessions() -> None:
|
||||
results = [
|
||||
{
|
||||
"prompt_id": f"DC-{index:02d}",
|
||||
"private_profile_id": f"profile-{index}",
|
||||
"persisted_session_id": f"session-{index}",
|
||||
"fresh_private_profile": True,
|
||||
"prior_prompt_ids": [],
|
||||
}
|
||||
for index in range(1, 7)
|
||||
]
|
||||
|
||||
assert suite.catalog_session_isolation(results) == {
|
||||
"all_profiles_unique": True,
|
||||
"dc_isolated": True,
|
||||
}
|
||||
results[-1]["persisted_session_id"] = results[0]["persisted_session_id"]
|
||||
assert suite.catalog_session_isolation(results)["dc_isolated"] is False
|
||||
|
||||
|
||||
def test_dummy_secret_never_leaks_from_executor_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
dummy = "DUMMY-SECRET-DO-NOT-LEAK"
|
||||
|
||||
def fake_run(command: list[str], **_kwargs: object) -> str:
|
||||
if command[0] == "gcloud":
|
||||
return dummy + "\n"
|
||||
raise RuntimeError(f"driver echoed {dummy}")
|
||||
|
||||
monkeypatch.setattr(suite.gcp, "run", fake_run)
|
||||
executor = suite.CloudSqlExecutor(executor_args())
|
||||
|
||||
with pytest.raises(RuntimeError) as captured:
|
||||
executor.read("select 1;")
|
||||
|
||||
assert dummy not in str(captured.value)
|
||||
assert "REDACTED_SECRET" in str(captured.value)
|
||||
|
||||
|
||||
def test_runtime_adapter_refuses_fallback_and_unclassified_writes() -> None:
|
||||
args = executor_args()
|
||||
executor = suite.CloudSqlExecutor(args)
|
||||
adapter = suite.RuntimeAdapter(
|
||||
args,
|
||||
executor,
|
||||
{
|
||||
"guard_snapshot": source_baseline_payload()["guard_snapshot"],
|
||||
"gate_schema": source_baseline_payload()["gate_schema"],
|
||||
},
|
||||
{"system_identifier": "system-1"},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="rebind"):
|
||||
adapter._psql_json("wrong-target", args.target_db, "select '{}'::jsonb;")
|
||||
with pytest.raises(RuntimeError, match="unclassified SQL write"):
|
||||
adapter._psql_json(adapter.target["id"], args.target_db, "delete from public.claims;")
|
||||
|
||||
|
||||
def test_runtime_adapter_restores_helpers_when_enter_fails(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
args = executor_args()
|
||||
executor = suite.CloudSqlExecutor(args)
|
||||
original = suite.bound._psql_json
|
||||
monkeypatch.setattr(executor, "json", lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("gate failed")))
|
||||
adapter = suite.RuntimeAdapter(
|
||||
args,
|
||||
executor,
|
||||
{
|
||||
"guard_snapshot": source_baseline_payload()["guard_snapshot"],
|
||||
"gate_schema": source_baseline_payload()["gate_schema"],
|
||||
},
|
||||
{"system_identifier": "system-1"},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="gate failed"):
|
||||
adapter.__enter__()
|
||||
|
||||
assert suite.bound._psql_json is original
|
||||
assert adapter.originals == []
|
||||
|
||||
|
||||
def test_lifecycle_wrapper_exposes_only_read_or_exact_stage_and_never_delivery(tmp_path: Path) -> None:
|
||||
identity_path, identity_sha = write_pinned_json(tmp_path, "identity.json", target_identity_payload())
|
||||
args = executor_args(
|
||||
cloudsql_tool=Path("hermes-agent/leoclean-bin/cloudsql_memory_tool.py").resolve(),
|
||||
target_identity_receipt=identity_path,
|
||||
target_identity_sha256=identity_sha,
|
||||
)
|
||||
|
||||
wrapper = suite._cloudsql_lifecycle_wrapper(
|
||||
args,
|
||||
tool_log=tmp_path / "calls.jsonl",
|
||||
run_marker="gcp-lifecycle-marker-1234",
|
||||
run_nonce="nonce",
|
||||
)
|
||||
|
||||
assert suite.lifecycle.STAGE_COMMAND in wrapper
|
||||
assert "accepts no model-supplied arguments" in wrapper
|
||||
assert "--stage-password-secret" in wrapper
|
||||
assert "--read-role" in wrapper
|
||||
assert 'user=$READ_ROLE' in wrapper
|
||||
assert "default_transaction_read_only=on" in wrapper
|
||||
assert "systemctl restart" not in wrapper
|
||||
assert "send_message" not in wrapper
|
||||
assert "approve-proposal" not in wrapper
|
||||
assert "apply-proposal" not in wrapper
|
||||
|
||||
catalog_wrapper = suite._build_gcp_read_wrapper(
|
||||
args,
|
||||
cloudsql_tool=args.cloudsql_tool,
|
||||
tool_log=tmp_path / "catalog.jsonl",
|
||||
run_nonce="catalog-nonce",
|
||||
prompt_id="DC-01",
|
||||
)
|
||||
assert catalog_wrapper.count('"prompt_id": "DC-01"') == 2
|
||||
assert '--user "$READ_ROLE"' in catalog_wrapper
|
||||
|
||||
|
||||
def test_capability_receipt_rejects_superuser_or_wrong_phase_acl() -> None:
|
||||
args = executor_args()
|
||||
receipt = json.loads(capability_receipt("kb_stage_writer", writable=True)[len(suite.ROLE_RECEIPT_PREFIX) :])
|
||||
suite.validate_capability_receipt(
|
||||
receipt,
|
||||
args=args,
|
||||
purpose="stage",
|
||||
role="kb_stage_writer",
|
||||
writable=True,
|
||||
)
|
||||
|
||||
receipt["is_superuser"] = True
|
||||
with pytest.raises(RuntimeError, match="not_superuser"):
|
||||
suite.validate_capability_receipt(
|
||||
receipt,
|
||||
args=args,
|
||||
purpose="stage",
|
||||
role="kb_stage_writer",
|
||||
writable=True,
|
||||
)
|
||||
receipt["is_superuser"] = False
|
||||
receipt["can_approve"] = True
|
||||
with pytest.raises(RuntimeError, match="can_approve"):
|
||||
suite.validate_capability_receipt(
|
||||
receipt,
|
||||
args=args,
|
||||
purpose="stage",
|
||||
role="kb_stage_writer",
|
||||
writable=True,
|
||||
)
|
||||
|
||||
|
||||
def test_validation_only_never_connects_or_calls_model_and_retains_report(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
args = argparse.Namespace(
|
||||
validation_only=True,
|
||||
target_db="teleo_clone_working_leo",
|
||||
cloudsql_tool=Path("hermes-agent/leoclean-bin/cloudsql_memory_tool.py"),
|
||||
manifest_sql=Path("ops/postgres_parity_manifest.sql"),
|
||||
parity_receipt=write_parity(tmp_path),
|
||||
output=output,
|
||||
)
|
||||
monkeypatch.setattr(suite.gcp, "database_identity", lambda *_args: pytest.fail("must not connect"))
|
||||
monkeypatch.setattr(suite.bound, "invoke_gateway_subprocess", lambda *_args: pytest.fail("must not call model"))
|
||||
|
||||
report = asyncio.run(suite.run_orchestrator(args))
|
||||
|
||||
assert report["status"] == "pass"
|
||||
assert report["database_connection_attempted"] is False
|
||||
assert report["model_call_attempted"] is False
|
||||
assert json.loads(output.read_text())["status"] == "pass"
|
||||
|
||||
|
||||
def test_failure_always_writes_structured_report(tmp_path: Path) -> None:
|
||||
output = tmp_path / "failure.json"
|
||||
bad_tool = tmp_path / "bad-tool.py"
|
||||
bad_tool.write_text("print('not reviewed')\n", encoding="utf-8")
|
||||
args = argparse.Namespace(
|
||||
validation_only=True,
|
||||
target_db="teleo_clone_working_leo",
|
||||
cloudsql_tool=bad_tool,
|
||||
manifest_sql=Path("ops/postgres_parity_manifest.sql"),
|
||||
parity_receipt=write_parity(tmp_path),
|
||||
output=output,
|
||||
)
|
||||
|
||||
report = asyncio.run(suite.run_orchestrator(args))
|
||||
|
||||
retained = json.loads(output.read_text())
|
||||
assert report["status"] == "fail"
|
||||
assert retained["status"] == "fail"
|
||||
assert retained["target_database"] == "teleo_clone_working_leo"
|
||||
assert retained["posted_to_telegram"] is False
|
||||
assert retained["systemd_restart_attempted"] is False
|
||||
assert retained["errors"][0]["phase"] == "orchestrator"
|
||||
|
||||
|
||||
def test_validation_only_cli_executes_end_to_end(tmp_path: Path) -> None:
|
||||
output = tmp_path / "cli-report.json"
|
||||
completed = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/run_gcp_generated_db_working_leo_suite.py",
|
||||
"--validation-only",
|
||||
"--target-db",
|
||||
"teleo_clone_working_leo",
|
||||
"--cloudsql-tool",
|
||||
"hermes-agent/leoclean-bin/cloudsql_memory_tool.py",
|
||||
"--manifest-sql",
|
||||
"ops/postgres_parity_manifest.sql",
|
||||
"--parity-receipt",
|
||||
str(write_parity(tmp_path)),
|
||||
"--output",
|
||||
str(output),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert json.loads(output.read_text(encoding="utf-8"))["status"] == "pass"
|
||||
assert '"mode": "validation_only"' in completed.stdout
|
||||
404
tests/test_gcp_iap_operator_access.py
Normal file
404
tests/test_gcp_iap_operator_access.py
Normal file
|
|
@ -0,0 +1,404 @@
|
|||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
PLAN = REPO_ROOT / "ops" / "plan_gcp_iap_operator_access.py"
|
||||
WORKFLOW = REPO_ROOT / ".github" / "workflows" / "gcp-iap-operator.yml"
|
||||
OPERATOR = REPO_ROOT / "scripts" / "gcp_iap_operator.sh"
|
||||
|
||||
|
||||
def run_plan(*args: str) -> str:
|
||||
return subprocess.check_output(["python3", str(PLAN), *args], cwd=REPO_ROOT, text=True)
|
||||
|
||||
|
||||
def test_plan_is_emit_only_and_pins_exact_github_identity() -> None:
|
||||
plan = json.loads(run_plan())
|
||||
|
||||
assert plan["mode"] == "dry_run_emit_only"
|
||||
assert plan["executes_commands"] is False
|
||||
condition = plan["wif"]["provider_condition"]
|
||||
assert "assertion.repository == 'living-ip/teleo-infrastructure'" in condition
|
||||
assert "assertion.ref == 'refs/heads/main'" in condition
|
||||
assert (
|
||||
"assertion.workflow_ref == "
|
||||
"'living-ip/teleo-infrastructure/.github/workflows/gcp-iap-operator.yml@refs/heads/main'"
|
||||
) in condition
|
||||
assert "assertion.event_name == 'workflow_dispatch'" in condition
|
||||
assert plan["wif"]["long_lived_google_credentials"] is False
|
||||
|
||||
|
||||
def test_plan_uses_exact_iap_firewall_and_least_privilege_split() -> None:
|
||||
plan = json.loads(run_plan())
|
||||
commands = "\n".join(plan["bootstrap_commands"])
|
||||
|
||||
assert plan["target"]["internal_ip"] == "10.60.0.3"
|
||||
assert plan["iap"]["condition"] == 'destination.ip == "10.60.0.3" && destination.port == 22'
|
||||
assert plan["iap"]["firewall"] == {
|
||||
"protocol_ports": "tcp:22",
|
||||
"rule": "teleo-prod-allow-ssh-iap",
|
||||
"source_range": "35.235.240.0/20",
|
||||
"target_tag": "teleo-prod-ssh",
|
||||
}
|
||||
assert plan["gcloud_ssh_custom_role"]["permissions"] == [
|
||||
"compute.instances.get",
|
||||
"compute.instances.list",
|
||||
"compute.projects.get",
|
||||
]
|
||||
assert plan["identities"]["status"]["os_login_role"] == "roles/compute.osLogin"
|
||||
assert plan["identities"]["status"]["sudo"] is False
|
||||
assert plan["identities"]["clone_operator"]["os_login_role"] == "roles/compute.osAdminLogin"
|
||||
assert plan["identities"]["clone_operator"]["operations"] == [
|
||||
"direct-claim-replay",
|
||||
"cleanup-clone",
|
||||
]
|
||||
assert "roles/iam.serviceAccountUser" in commands
|
||||
assert "--ssh-key-expire-after=5m" in commands
|
||||
assert "--ssh-key-expiration" not in commands
|
||||
assert "sha256sum -c dispatcher.sha256" in commands
|
||||
assert "teleo-gcp-iap-operator-v1" in commands
|
||||
for role in plan["explicitly_absent"]["roles"]:
|
||||
assert f"--role {role}" not in commands
|
||||
assert "compute.instances.setMetadata" not in plan["gcloud_ssh_custom_role"]["permissions"]
|
||||
|
||||
|
||||
def test_public_rule_is_disabled_only_after_workflow_status_verification() -> None:
|
||||
plan = json.loads(run_plan())
|
||||
verification = plan["post_bootstrap_verification"]
|
||||
joined = "\n".join(verification)
|
||||
|
||||
dispatch_index = next(i for i, command in enumerate(verification) if "gh workflow run" in command)
|
||||
disable_index = next(
|
||||
i
|
||||
for i, command in enumerate(verification)
|
||||
if "teleo-prod-allow-ssh-current-ip" in command and "--disabled" in command
|
||||
)
|
||||
assert dispatch_index < disable_index
|
||||
assert "operation=status" in joined
|
||||
assert "target_db=teleo_clone_status" in joined
|
||||
assert "dispatcher_sha256" in joined
|
||||
assert 'Path("scripts/gcp_iap_operator.sh")' in joined
|
||||
assert plan["public_ingress_cutover"]["automatic_fallback_to_public_ssh"] is False
|
||||
assert "DATA_READ for iap.googleapis.com" in plan["audit_logging"]["guidance"]
|
||||
assert plan["revocation_commands"]
|
||||
|
||||
|
||||
def test_shell_plan_is_clearly_dry_run() -> None:
|
||||
shell = run_plan("--format", "shell")
|
||||
|
||||
assert shell.startswith("# DRY RUN:")
|
||||
assert "# bootstrap_commands" in shell
|
||||
assert "# post_bootstrap_verification" in shell
|
||||
assert "# revocation_commands" in shell
|
||||
assert "subprocess" not in PLAN.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def load_workflow() -> dict[str, object]:
|
||||
return yaml.load(WORKFLOW.read_text(encoding="utf-8"), Loader=yaml.BaseLoader)
|
||||
|
||||
|
||||
def test_workflow_accepts_only_fixed_operation_and_bounded_ids() -> None:
|
||||
workflow = load_workflow()
|
||||
inputs = workflow["on"]["workflow_dispatch"]["inputs"]
|
||||
|
||||
assert set(inputs) == {"operation", "request_id", "target_db"}
|
||||
assert inputs["operation"]["options"] == ["status", "direct-claim-replay", "cleanup-clone"]
|
||||
assert inputs["target_db"]["default"] == "teleo_clone_status"
|
||||
source = WORKFLOW.read_text(encoding="utf-8")
|
||||
assert '[[ "${GITHUB_REF}" == "refs/heads/main" ]]' in source
|
||||
assert '[[ "${GITHUB_WORKFLOW_REF}" == "${EXPECTED_WORKFLOW_REF}" ]]' in source
|
||||
assert "request_id_re='^iap-[a-z0-9]{12,32}$'" in source
|
||||
assert "target_db_re='^teleo_clone_[a-z0-9][a-z0-9_]{0,50}$'" in source
|
||||
for forbidden_input in ("command:", "path:", "host:", "project:", "zone:", "ip:", "ref:", "service_account:"):
|
||||
assert forbidden_input not in inputs
|
||||
|
||||
|
||||
def test_workflow_uses_wif_official_actions_and_sanitized_short_retention() -> None:
|
||||
workflow = load_workflow()
|
||||
assert workflow["permissions"] == {"contents": "read", "id-token": "write"}
|
||||
steps = workflow["jobs"]["operate"]["steps"]
|
||||
actions = [step["uses"] for step in steps if "uses" in step]
|
||||
|
||||
assert actions == [
|
||||
"actions/checkout@v4",
|
||||
"google-github-actions/auth@v3",
|
||||
"google-github-actions/setup-gcloud@v3",
|
||||
"actions/upload-artifact@v4",
|
||||
]
|
||||
auth = next(step for step in steps if step.get("uses") == "google-github-actions/auth@v3")
|
||||
assert auth["with"]["create_credentials_file"] == "true"
|
||||
assert auth["with"]["cleanup_credentials"] == "true"
|
||||
upload = next(step for step in steps if step.get("uses") == "actions/upload-artifact@v4")
|
||||
assert upload["with"]["path"] == "${{ env.RESULT_DIR }}/result.json"
|
||||
assert upload["with"]["retention-days"] == "7"
|
||||
validate_index = next(i for i, step in enumerate(steps) if step.get("id") == "validate")
|
||||
auth_index = next(i for i, step in enumerate(steps) if step.get("uses") == "google-github-actions/auth@v3")
|
||||
assert validate_index < auth_index
|
||||
|
||||
|
||||
def test_workflow_builds_exact_main_bundle_from_tracked_files_and_hashes() -> None:
|
||||
source = WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
expected_files = [
|
||||
"scripts/gcp_iap_operator.sh",
|
||||
"scripts/run_gcp_generated_db_direct_claim_suite.py",
|
||||
"scripts/run_leo_clone_bound_handler_checkpoint.py",
|
||||
"scripts/working_leo_open_ended_benchmark.py",
|
||||
"hermes-agent/leoclean-bin/cloudsql_memory_tool.py",
|
||||
"ops/postgres_parity_manifest.sql",
|
||||
]
|
||||
for relative in expected_files:
|
||||
assert f'"{relative}"' in source
|
||||
assert 'f"docs/reports/leo-working-state-20260709/{target_db}-canonical-parity-receipt.json"' in source
|
||||
assert '["git", "ls-files", "--error-unmatch", "--", relative]' in source
|
||||
assert 'if head != os.environ["GITHUB_SHA"]' in source
|
||||
assert "hashlib.sha256(data).hexdigest()" in source
|
||||
assert '"schema": "livingip.gcpIapOperatorBundle.v1"' in source
|
||||
assert 'bundle_path = bundle_dir / f"{request_id}.tar.gz"' in source
|
||||
|
||||
|
||||
def test_operator_allowlists_commands_and_has_no_live_mutation_surface() -> None:
|
||||
source = OPERATOR.read_text(encoding="utf-8")
|
||||
|
||||
assert "status | direct-claim-replay | cleanup-clone" in source
|
||||
assert "scripts/run_gcp_generated_db_direct_claim_suite.py" in source
|
||||
assert "six_replies_returned" in source
|
||||
assert "posted_to_telegram" in source
|
||||
assert "live_service_unchanged" in source
|
||||
assert "live_profile_unchanged" in source
|
||||
assert "%s-canonical-parity-receipt.json" in source
|
||||
assert "/opt/teleo-eval" not in source
|
||||
assert "systemctl restart" not in source
|
||||
assert "systemctl stop" not in source
|
||||
assert "systemctl start" not in source
|
||||
assert "curl " not in source
|
||||
assert "--profile" not in source
|
||||
assert "eval " not in source
|
||||
|
||||
|
||||
def test_operator_cleanup_is_bound_to_marker_clone_prefix_and_exact_run_dir() -> None:
|
||||
source = OPERATOR.read_text(encoding="utf-8")
|
||||
|
||||
assert "TARGET_DB_RE='^teleo_clone_" in source
|
||||
assert 'expected="$REMOTE_RUN_ROOT/$request_id"' in source
|
||||
assert '[[ "$resolved" == "$expected" ]]' in source
|
||||
assert ".run-owner" in source
|
||||
assert "run ownership marker does not match request_id and target_db" in source
|
||||
assert 'drop database :"target_db";' in source
|
||||
assert '[[ "$remaining" == "0" ]]' in source
|
||||
assert 'rm -rf --one-file-system -- "$run_dir"' in source
|
||||
|
||||
|
||||
def test_operator_uses_five_minute_key_private_modes_and_cleanup_trap() -> None:
|
||||
source = OPERATOR.read_text(encoding="utf-8")
|
||||
|
||||
assert '--ssh-key-file="$ssh_key"' in source
|
||||
assert "--ssh-key-expire-after=5m" in source
|
||||
assert "--ssh-key-expiration" not in source
|
||||
assert "chmod 0700" in source
|
||||
assert "chmod 0600" in source
|
||||
assert "trap cleanup_runner_temp EXIT" in source
|
||||
assert "--tunnel-through-iap" in source
|
||||
assert "gcloud compute scp" in source
|
||||
assert "--verbosity=debug" not in source
|
||||
|
||||
|
||||
def test_dispatcher_validates_exact_members_hashes_context_and_self_hash() -> None:
|
||||
source = OPERATOR.read_text(encoding="utf-8")
|
||||
|
||||
assert 'expected_members = sorted([*expected, "bundle-manifest.json"])' in source
|
||||
assert "bundle member allowlist mismatch" in source
|
||||
assert "bundle manifest file allowlist mismatch" in source
|
||||
assert 'receipt.get("sha256") != hashlib.sha256(data).hexdigest()' in source
|
||||
assert '"workflow_ref": expected_workflow_ref' in source
|
||||
assert "installed dispatcher does not match the reviewed main bundle" in source
|
||||
assert 'dispatcher_sha != files["scripts/gcp_iap_operator.sh"]["sha256"]' in source
|
||||
|
||||
|
||||
def test_runner_fake_gcloud_smoke_retains_only_sanitized_result(tmp_path: Path) -> None:
|
||||
bin_dir = tmp_path / "bin"
|
||||
runner_temp = tmp_path / "runner"
|
||||
result_dir = runner_temp / "result"
|
||||
invocation = tmp_path / "gcloud-argv.txt"
|
||||
bin_dir.mkdir()
|
||||
runner_temp.mkdir()
|
||||
fake_gcloud = bin_dir / "gcloud"
|
||||
fake_gcloud.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
'printf \'%s\\n\' "$@" > "${FAKE_GCLOUD_ARGV}"\n'
|
||||
"printf '%s\\n' "
|
||||
'\'{"operation":"status","request_id":"iap-abcdefghijkl",\''
|
||||
'\'"target_db":"teleo_clone_status","status":"pass",\''
|
||||
'\'"instance":"teleo-prod-1","expected_internal_ip":"10.60.0.3",\''
|
||||
'\'"active_state":"active","sub_state":"running","nrestarts":0,\''
|
||||
'\'"dispatcher_version":"teleo-gcp-iap-operator-v1",\''
|
||||
'\'"dispatcher_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}\'\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_gcloud.chmod(0o755)
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||
"RUNNER_TEMP": str(runner_temp),
|
||||
"RESULT_DIR": str(result_dir),
|
||||
"FAKE_GCLOUD_ARGV": str(invocation),
|
||||
}
|
||||
|
||||
completed = subprocess.run(
|
||||
["bash", str(OPERATOR), "status", "iap-abcdefghijkl", "teleo_clone_status"],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
result = json.loads((result_dir / "result.json").read_text(encoding="utf-8"))
|
||||
assert result["status"] == "pass"
|
||||
assert result["credential_values_logged"] is False
|
||||
assert result["raw_stdout_retained"] is False
|
||||
assert result["raw_stderr_retained"] is False
|
||||
assert result["remote_result"]["active_state"] == "active"
|
||||
argv = invocation.read_text(encoding="utf-8")
|
||||
assert "compute\nssh\nteleo-prod-1\n" in argv
|
||||
assert "--tunnel-through-iap" in argv
|
||||
assert "--ssh-key-expire-after=5m" in argv
|
||||
assert not list(runner_temp.glob("teleo-iap-operator.*"))
|
||||
|
||||
|
||||
def test_runner_fails_closed_when_remote_output_is_not_valid_json(tmp_path: Path) -> None:
|
||||
bin_dir = tmp_path / "bin"
|
||||
runner_temp = tmp_path / "runner"
|
||||
result_dir = runner_temp / "result"
|
||||
bin_dir.mkdir()
|
||||
runner_temp.mkdir()
|
||||
fake_gcloud = bin_dir / "gcloud"
|
||||
fake_gcloud.write_text("#!/usr/bin/env bash\nprintf 'not-json\\n'\n", encoding="utf-8")
|
||||
fake_gcloud.chmod(0o755)
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||
"RUNNER_TEMP": str(runner_temp),
|
||||
"RESULT_DIR": str(result_dir),
|
||||
}
|
||||
|
||||
completed = subprocess.run(
|
||||
["bash", str(OPERATOR), "status", "iap-abcdefghijkl", "teleo_clone_status"],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert completed.returncode == 90
|
||||
result = json.loads((result_dir / "result.json").read_text(encoding="utf-8"))
|
||||
assert result["status"] == "fail"
|
||||
assert result["remote_result"] == {}
|
||||
assert not list(runner_temp.glob("teleo-iap-operator.*"))
|
||||
|
||||
|
||||
def test_clone_runner_scps_only_request_derived_bundle_over_iap(tmp_path: Path) -> None:
|
||||
bin_dir = tmp_path / "bin"
|
||||
runner_temp = tmp_path / "runner"
|
||||
result_dir = runner_temp / "result"
|
||||
bundle_dir = runner_temp / "gcp-iap-operator-bundle"
|
||||
invocation = tmp_path / "gcloud-argv.txt"
|
||||
bin_dir.mkdir()
|
||||
runner_temp.mkdir()
|
||||
bundle_dir.mkdir(mode=0o700)
|
||||
bundle = bundle_dir / "iap-abcdefghijkl.tar.gz"
|
||||
bundle.write_bytes(b"reviewed-bundle")
|
||||
bundle.chmod(0o600)
|
||||
fake_gcloud = bin_dir / "gcloud"
|
||||
fake_gcloud.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
'printf \'%s \' "$@" >> "${FAKE_GCLOUD_ARGV}"\n'
|
||||
"printf '\\n' >> \"${FAKE_GCLOUD_ARGV}\"\n"
|
||||
'if [[ "$2" == "ssh" ]]; then\n'
|
||||
" printf '%s\\n' "
|
||||
'\'{"operation":"direct-claim-replay","request_id":"iap-abcdefghijkl",\''
|
||||
'\'"target_db":"teleo_clone_test","status":"pass",\''
|
||||
'\'"bundle_commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",\''
|
||||
'\'"bundle_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}\'\n'
|
||||
"fi\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_gcloud.chmod(0o755)
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||
"RUNNER_TEMP": str(runner_temp),
|
||||
"RESULT_DIR": str(result_dir),
|
||||
"FAKE_GCLOUD_ARGV": str(invocation),
|
||||
}
|
||||
|
||||
completed = subprocess.run(
|
||||
["bash", str(OPERATOR), "direct-claim-replay", "iap-abcdefghijkl", "teleo_clone_test"],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
calls = invocation.read_text(encoding="utf-8").splitlines()
|
||||
assert len(calls) == 2
|
||||
assert "compute scp" in calls[0]
|
||||
assert str(bundle) in calls[0]
|
||||
assert "teleo-prod-1:.teleo-iap-upload-iap-abcdefghijkl.tar.gz" in calls[0]
|
||||
assert "--tunnel-through-iap" in calls[0]
|
||||
assert "--ssh-key-expire-after=5m" in calls[0]
|
||||
assert "compute ssh" in calls[1]
|
||||
assert "--tunnel-through-iap" in calls[1]
|
||||
|
||||
|
||||
def test_clone_runner_rejects_bundle_symlink_before_gcloud(tmp_path: Path) -> None:
|
||||
bin_dir = tmp_path / "bin"
|
||||
runner_temp = tmp_path / "runner"
|
||||
result_dir = runner_temp / "result"
|
||||
bundle_dir = runner_temp / "gcp-iap-operator-bundle"
|
||||
outside = tmp_path / "outside.tar.gz"
|
||||
marker = tmp_path / "gcloud-called"
|
||||
bin_dir.mkdir()
|
||||
runner_temp.mkdir()
|
||||
bundle_dir.mkdir(mode=0o700)
|
||||
outside.write_bytes(b"outside")
|
||||
(bundle_dir / "iap-abcdefghijkl.tar.gz").symlink_to(outside)
|
||||
fake_gcloud = bin_dir / "gcloud"
|
||||
fake_gcloud.write_text(
|
||||
'#!/usr/bin/env bash\ntouch "${FAKE_GCLOUD_MARKER}"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_gcloud.chmod(0o755)
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||
"RUNNER_TEMP": str(runner_temp),
|
||||
"RESULT_DIR": str(result_dir),
|
||||
"FAKE_GCLOUD_MARKER": str(marker),
|
||||
}
|
||||
|
||||
completed = subprocess.run(
|
||||
["bash", str(OPERATOR), "cleanup-clone", "iap-abcdefghijkl", "teleo_clone_test"],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert completed.returncode == 2
|
||||
assert "fixed workflow bundle is absent or unsafe" in completed.stderr
|
||||
assert not marker.exists()
|
||||
|
||||
|
||||
def test_google_auth_credential_file_pattern_is_ignored_exactly() -> None:
|
||||
ignore_lines = (REPO_ROOT / ".gitignore").read_text(encoding="utf-8").splitlines()
|
||||
|
||||
assert "gha-creds-*.json" in ignore_lines
|
||||
Loading…
Reference in a new issue