Add authenticated GCP Observatory read adapter (#154)
Some checks are pending
CI / lint-and-test (push) Waiting to run
Some checks are pending
CI / lint-and-test (push) Waiting to run
This commit is contained in:
parent
15c30f1fbe
commit
92da54d804
13 changed files with 1559 additions and 1 deletions
199
.github/workflows/gcp-observatory-read-adapter.yml
vendored
Normal file
199
.github/workflows/gcp-observatory-read-adapter.yml
vendored
Normal file
|
|
@ -0,0 +1,199 @@
|
||||||
|
name: gcp-observatory-read-adapter
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
action:
|
||||||
|
description: Build only, or deploy the protected staging service and run live receipts
|
||||||
|
required: true
|
||||||
|
default: build_only
|
||||||
|
type: choice
|
||||||
|
options:
|
||||||
|
- build_only
|
||||||
|
- deploy_staging
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
id-token: write
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: gcp-observatory-read-adapter-${{ github.ref }}
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
env:
|
||||||
|
PROJECT_ID: teleo-501523
|
||||||
|
REGION: europe-west6
|
||||||
|
ZONE: europe-west6-a
|
||||||
|
REPOSITORY: teleo
|
||||||
|
IMAGE_NAME: observatory-read-adapter
|
||||||
|
SERVICE_NAME: observatory-read-adapter-staging
|
||||||
|
RUNTIME_SERVICE_ACCOUNT: sa-observatory-read-adapter@teleo-501523.iam.gserviceaccount.com
|
||||||
|
DB_IAM_USER: sa-observatory-read-adapter@teleo-501523.iam
|
||||||
|
API_KEY_SECRET: observatory-read-api-key-staging
|
||||||
|
WORKLOAD_IDENTITY_PROVIDER: projects/785938879453/locations/global/workloadIdentityPools/github-actions/providers/living-ip-github
|
||||||
|
DEPLOY_SERVICE_ACCOUNT: sa-artifact-builder@teleo-501523.iam.gserviceaccount.com
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: Test, build, and push adapter image
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 20
|
||||||
|
outputs:
|
||||||
|
image_ref: ${{ steps.image.outputs.image_ref }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.11"
|
||||||
|
cache: pip
|
||||||
|
|
||||||
|
- name: Run focused adapter tests
|
||||||
|
run: |
|
||||||
|
python -m pip install -e '.[dev]'
|
||||||
|
python -m pytest tests/test_observatory_read_adapter.py -q
|
||||||
|
python -m ruff check observatory_read_adapter tests/test_observatory_read_adapter.py
|
||||||
|
|
||||||
|
- id: auth
|
||||||
|
uses: google-github-actions/auth@v3
|
||||||
|
with:
|
||||||
|
workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}
|
||||||
|
service_account: ${{ env.DEPLOY_SERVICE_ACCOUNT }}
|
||||||
|
|
||||||
|
- uses: google-github-actions/setup-gcloud@v3
|
||||||
|
with:
|
||||||
|
project_id: ${{ env.PROJECT_ID }}
|
||||||
|
|
||||||
|
- name: Configure Artifact Registry
|
||||||
|
run: gcloud auth configure-docker "${REGION}-docker.pkg.dev" --quiet
|
||||||
|
|
||||||
|
- id: image
|
||||||
|
name: Build, smoke, and push immutable image
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
tag="${GITHUB_SHA::12}"
|
||||||
|
image="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPOSITORY}/${IMAGE_NAME}:${tag}"
|
||||||
|
docker build -f Dockerfile.observatory-read-adapter -t "${image}" .
|
||||||
|
docker run --rm --env PYTHONPYCACHEPREFIX=/tmp/pycache \
|
||||||
|
"${image}" python -m compileall -q /app/observatory_read_adapter
|
||||||
|
docker run --rm "${image}" python -c 'import observatory_read_adapter; print("adapter-import-ok")'
|
||||||
|
docker push "${image}" | tee docker-push.log
|
||||||
|
digest="$(awk '/digest: sha256:/ {print $3}' docker-push.log | tail -1)"
|
||||||
|
test -n "${digest}"
|
||||||
|
image_ref="${image}@${digest}"
|
||||||
|
printf 'image_ref=%s\n' "${image_ref}" >> "${GITHUB_OUTPUT}"
|
||||||
|
printf 'revision=%s\nimage_ref=%s\n' "${GITHUB_SHA}" "${image_ref}" > observatory-adapter-image.txt
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: observatory-adapter-image
|
||||||
|
path: observatory-adapter-image.txt
|
||||||
|
if-no-files-found: error
|
||||||
|
|
||||||
|
deploy-staging:
|
||||||
|
name: Deploy and prove staging read path
|
||||||
|
if: ${{ inputs.action == 'deploy_staging' }}
|
||||||
|
needs: build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 20
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- id: auth
|
||||||
|
uses: google-github-actions/auth@v3
|
||||||
|
with:
|
||||||
|
workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}
|
||||||
|
service_account: ${{ env.DEPLOY_SERVICE_ACCOUNT }}
|
||||||
|
|
||||||
|
- uses: google-github-actions/setup-gcloud@v3
|
||||||
|
with:
|
||||||
|
project_id: ${{ env.PROJECT_ID }}
|
||||||
|
|
||||||
|
- name: Deploy bounded GCP staging service
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
IMAGE_REF: ${{ needs.build.outputs.image_ref }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
gcloud run deploy "${SERVICE_NAME}" \
|
||||||
|
--project="${PROJECT_ID}" \
|
||||||
|
--region="${REGION}" \
|
||||||
|
--image="${IMAGE_REF}" \
|
||||||
|
--execution-environment=gen2 \
|
||||||
|
--service-account="${RUNTIME_SERVICE_ACCOUNT}" \
|
||||||
|
--network=teleo-staging-net \
|
||||||
|
--subnet=teleo-staging-europe-west6 \
|
||||||
|
--vpc-egress=private-ranges-only \
|
||||||
|
--ingress=all \
|
||||||
|
--allow-unauthenticated \
|
||||||
|
--port=8080 \
|
||||||
|
--cpu=1 \
|
||||||
|
--memory=512Mi \
|
||||||
|
--concurrency=8 \
|
||||||
|
--max-instances=2 \
|
||||||
|
--timeout=15 \
|
||||||
|
--set-secrets="OBSERVATORY_API_KEY=${API_KEY_SECRET}:latest" \
|
||||||
|
--set-env-vars="GCP_PROJECT_ID=${PROJECT_ID},CLOUD_SQL_INSTANCE=${PROJECT_ID}:${REGION}:teleo-pgvector-standby,DB_NAME=teleo_canonical,DB_IAM_USER=${DB_IAM_USER},DB_AUTHORIZATION_ROLE=kb_observatory_read,SERVICE_REVISION=${GITHUB_SHA}" \
|
||||||
|
--labels="livingip-tier=gcp-staging,livingip-surface=observatory-read-adapter" \
|
||||||
|
--quiet
|
||||||
|
|
||||||
|
- name: Capture positive and negative live receipts
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
service_url="$(gcloud run services describe "${SERVICE_NAME}" --project="${PROJECT_ID}" --region="${REGION}" --format='value(status.url)')"
|
||||||
|
revision="$(gcloud run services describe "${SERVICE_NAME}" --project="${PROJECT_ID}" --region="${REGION}" --format='value(status.latestReadyRevisionName)')"
|
||||||
|
api_key="$(gcloud secrets versions access latest --secret="${API_KEY_SECRET}" --project="${PROJECT_ID}")"
|
||||||
|
test -n "${service_url}"
|
||||||
|
test -n "${revision}"
|
||||||
|
test -n "${api_key}"
|
||||||
|
|
||||||
|
curl --fail --silent --show-error \
|
||||||
|
--header "X-Api-Key: ${api_key}" \
|
||||||
|
"${service_url}/v1/claims/sample" > positive.json
|
||||||
|
jq -e '
|
||||||
|
.schema == "livingip.observatory-canonical-claim.v1"
|
||||||
|
and .read_only == true
|
||||||
|
and .provenance.database == "teleo_canonical"
|
||||||
|
and .provenance.authorization_role == "kb_observatory_read"
|
||||||
|
and .provenance.transaction_read_only == true
|
||||||
|
and .provenance.write_privileges_denied == true
|
||||||
|
and (.canonical.evidence | length) > 0
|
||||||
|
and .proposal_ledger.distinct_from_canonical == true
|
||||||
|
' positive.json >/dev/null
|
||||||
|
|
||||||
|
anonymous_status="$(curl --silent --output anonymous.json --write-out '%{http_code}' "${service_url}/v1/claims/sample")"
|
||||||
|
write_status="$(curl --silent --output write.json --write-out '%{http_code}' \
|
||||||
|
--request POST --header "X-Api-Key: ${api_key}" --header 'Content-Type: application/json' \
|
||||||
|
--data '{"status":"applied"}' "${service_url}/v1/claims/sample")"
|
||||||
|
test "${anonymous_status}" = 401
|
||||||
|
test "${write_status}" = 405
|
||||||
|
|
||||||
|
jq -n \
|
||||||
|
--arg service_url "${service_url}" \
|
||||||
|
--arg revision "${revision}" \
|
||||||
|
--arg git_sha "${GITHUB_SHA}" \
|
||||||
|
--argjson positive "$(cat positive.json)" \
|
||||||
|
--arg anonymous_status "${anonymous_status}" \
|
||||||
|
--arg write_status "${write_status}" \
|
||||||
|
'{
|
||||||
|
schema: "livingip.observatory-read-adapter-live-receipt.v1",
|
||||||
|
required_tier: "T3_live_readonly",
|
||||||
|
service_url: $service_url,
|
||||||
|
revision: $revision,
|
||||||
|
git_sha: $git_sha,
|
||||||
|
positive: $positive,
|
||||||
|
negative: {
|
||||||
|
anonymous_http_status: ($anonymous_status | tonumber),
|
||||||
|
authenticated_post_http_status: ($write_status | tonumber)
|
||||||
|
},
|
||||||
|
production_repoint_executed: false
|
||||||
|
}' > observatory-read-adapter-live-receipt.json
|
||||||
|
unset api_key
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: observatory-read-adapter-live-receipt
|
||||||
|
path: observatory-read-adapter-live-receipt.json
|
||||||
|
if-no-files-found: error
|
||||||
20
Dockerfile.observatory-read-adapter
Normal file
20
Dockerfile.observatory-read-adapter
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
FROM python:3.11-slim
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1
|
||||||
|
ENV PYTHONUNBUFFERED=1
|
||||||
|
ENV PORT=8080
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY pyproject.toml README.md /app/
|
||||||
|
COPY observatory_read_adapter /app/observatory_read_adapter
|
||||||
|
COPY lib /app/lib
|
||||||
|
|
||||||
|
RUN pip install --no-cache-dir --upgrade pip \
|
||||||
|
&& pip install --no-cache-dir ".[observatory]" \
|
||||||
|
&& addgroup --system --gid 10001 adapter \
|
||||||
|
&& adduser --system --uid 10001 --ingroup adapter --no-create-home adapter
|
||||||
|
|
||||||
|
USER 10001:10001
|
||||||
|
|
||||||
|
CMD ["python", "-m", "observatory_read_adapter"]
|
||||||
102
docs/observatory-read-adapter-gcp-staging.md
Normal file
102
docs/observatory-read-adapter-gcp-staging.md
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
# GCP Observatory Read Adapter
|
||||||
|
|
||||||
|
## Definition of Working
|
||||||
|
|
||||||
|
- Working target: an authenticated GCP staging API returns one canonical claim
|
||||||
|
with source/evidence provenance and a separately labeled proposal ledger.
|
||||||
|
- Operator path: `GET /v1/claims/sample` or `GET /v1/claims/{uuid}` with the
|
||||||
|
server-only `X-Api-Key` value.
|
||||||
|
- Done means: the response names `teleo_canonical`, the dedicated IAM database
|
||||||
|
principal and `kb_observatory_read`, reports a read-only transaction and
|
||||||
|
denied write privileges, and live anonymous/write HTTP attempts return
|
||||||
|
`401`/`405`.
|
||||||
|
- Not done: the VPS `teleo-pg`, a public Cloud SQL address, a browser-visible
|
||||||
|
key, a mock-only receipt, or a production Vercel repoint.
|
||||||
|
- Required tier: `T3_live_readonly`.
|
||||||
|
|
||||||
|
## Existing Connector Boundary
|
||||||
|
|
||||||
|
The current `living-ip/livingip-web` main branch reads claims through
|
||||||
|
`src/lib/api.ts`. It selects `NEXT_PUBLIC_API_URL` and otherwise falls back to
|
||||||
|
`http://77.42.65.182:8081`, then calls the legacy `/api/claims` routes. That is
|
||||||
|
the VPS-local/public Observatory path. It is not safe to replace that public
|
||||||
|
environment variable with this protected API: the response shape differs and
|
||||||
|
the adapter key must never enter a `NEXT_PUBLIC_` variable or browser bundle.
|
||||||
|
|
||||||
|
The existing Teleo route `GET /api/kb/claims` is the compatibility reference
|
||||||
|
for route-scoped authentication, but its default database/container settings
|
||||||
|
still describe the VPS-local process. This service is a separate GCP runtime;
|
||||||
|
it does not change Argus or PR #148's Leo runtime files.
|
||||||
|
|
||||||
|
## Runtime Contract
|
||||||
|
|
||||||
|
- Cloud Run service: `observatory-read-adapter-staging` in `europe-west6`.
|
||||||
|
- Direct VPC egress: `teleo-staging-net` / `teleo-staging-europe-west6`.
|
||||||
|
- Cloud SQL: `teleo-501523:europe-west6:teleo-pgvector-standby`, private IP.
|
||||||
|
- Database: `teleo_canonical` only.
|
||||||
|
- Runtime identity: `sa-observatory-read-adapter@teleo-501523.iam.gserviceaccount.com`.
|
||||||
|
- IAM database user: `sa-observatory-read-adapter@teleo-501523.iam`.
|
||||||
|
- Database authorization role: `kb_observatory_read` (`NOLOGIN`).
|
||||||
|
- API authentication: route-scoped key from Secret Manager secret
|
||||||
|
`observatory-read-api-key-staging`; the value is not in Git or deploy logs.
|
||||||
|
- No mutation routes exist. Authenticated unsupported methods return `405`.
|
||||||
|
|
||||||
|
The Cloud SQL Python Connector uses Application Default Credentials from the
|
||||||
|
Cloud Run service identity, automatic IAM database authentication, and private
|
||||||
|
IP. Every request starts a read-only transaction and refuses the response if
|
||||||
|
the database, IAM principal, authorization membership, required reads, or
|
||||||
|
effective write-denial checks drift.
|
||||||
|
|
||||||
|
## One-Time Staging Bootstrap
|
||||||
|
|
||||||
|
These are staging mutations. Run them only with an authenticated GCP operator;
|
||||||
|
they do not route production traffic.
|
||||||
|
|
||||||
|
1. Create the dedicated runtime service account and grant only
|
||||||
|
`roles/cloudsql.client` and `roles/cloudsql.instanceUser` in project
|
||||||
|
`teleo-501523`.
|
||||||
|
2. Add the service account as a Cloud SQL IAM service-account user on
|
||||||
|
`teleo-pgvector-standby`.
|
||||||
|
3. From the existing private GCP VM database-admin path, run
|
||||||
|
`ops/observatory_read_role.sql` against `teleo_canonical` with
|
||||||
|
`OBSERVATORY_DB_IAM_USER=sa-observatory-read-adapter@teleo-501523.iam`.
|
||||||
|
Do not read, copy, or retain the administrator password.
|
||||||
|
4. Create `observatory-read-api-key-staging`, add a generated 32-byte-or-longer
|
||||||
|
value through stdin, and grant that secret's accessor role only to the
|
||||||
|
runtime service account and the bounded staging canary identity.
|
||||||
|
5. Dispatch `.github/workflows/gcp-observatory-read-adapter.yml` with
|
||||||
|
`action=deploy_staging` from the exact reviewed branch revision.
|
||||||
|
|
||||||
|
The workflow builds an immutable image, deploys at most two Cloud Run
|
||||||
|
instances, and retains the positive response plus anonymous `401` and
|
||||||
|
authenticated POST `405` receipts. It does not modify Vercel.
|
||||||
|
|
||||||
|
## Unexecuted Cutover Packet
|
||||||
|
|
||||||
|
Production cutover requires a separate exact authorization and a
|
||||||
|
`living-ip/livingip-web` change. Do not execute these steps from this lane.
|
||||||
|
|
||||||
|
1. Add server-only Vercel secrets `OBSERVATORY_CANONICAL_API_URL` and
|
||||||
|
`OBSERVATORY_CANONICAL_API_KEY` to a protected preview environment. The URL
|
||||||
|
targets the GCP service; neither setting may use `NEXT_PUBLIC_`.
|
||||||
|
2. Add a Next.js server route or server action that calls the GCP API, validates
|
||||||
|
`livingip.observatory-canonical-claim.v1`, and maps it to the Observatory UI.
|
||||||
|
The browser calls only the same-origin Next.js route.
|
||||||
|
3. Prove preview Deployment Protection, anonymous denial, source/evidence
|
||||||
|
rendering, proposal/live labels, and no browser key exposure.
|
||||||
|
4. Obtain exact production-repoint authorization naming the reviewed
|
||||||
|
`livingip-web` commit and Vercel project before changing production values.
|
||||||
|
|
||||||
|
## Unexecuted Rollback Packet
|
||||||
|
|
||||||
|
1. Revert the authorized `livingip-web` connector commit or disable its
|
||||||
|
server-side feature flag.
|
||||||
|
2. Restore the prior production Vercel values and redeploy the last accepted
|
||||||
|
production revision.
|
||||||
|
3. Verify `/claims` again uses the prior `/api/claims` contract.
|
||||||
|
4. Leave the private Cloud SQL instance and adapter role intact while receipts
|
||||||
|
are reviewed; scale the staging Cloud Run service to zero if isolation is
|
||||||
|
required.
|
||||||
|
5. Delete the staging service/role/secret only under a separate cleanup
|
||||||
|
decision. Rollback never adds a Cloud SQL public IP and never redirects the
|
||||||
|
adapter to VPS `teleo-pg`.
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
{
|
||||||
|
"fallback_goal_path": "goal.md",
|
||||||
|
"platform_goal_after": {
|
||||||
|
"objective": "Complete and verify the private GCP Observatory read adapter defined in this goal file, stopping before production repoint without exact authorization.",
|
||||||
|
"status": "active",
|
||||||
|
"thread_id": "019f6350-69fa-71f3-944b-3fc0065f4fec"
|
||||||
|
},
|
||||||
|
"platform_goal_before": null,
|
||||||
|
"platform_goal_repair_action": "create_goal",
|
||||||
|
"platform_goal_replaced": false,
|
||||||
|
"why_not_replaced": "No prior platform Goal existed; a new active Goal was created from goal.md."
|
||||||
|
}
|
||||||
6
observatory_read_adapter/__init__.py
Normal file
6
observatory_read_adapter/__init__.py
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
"""Authenticated, read-only Observatory adapter for canonical Teleo claims."""
|
||||||
|
|
||||||
|
from .app import create_app
|
||||||
|
from .config import Settings
|
||||||
|
|
||||||
|
__all__ = ["Settings", "create_app"]
|
||||||
18
observatory_read_adapter/__main__.py
Normal file
18
observatory_read_adapter/__main__.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from aiohttp import web
|
||||||
|
|
||||||
|
from .app import create_app
|
||||||
|
from .config import Settings
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
settings = Settings.from_env()
|
||||||
|
web.run_app(create_app(settings), host="0.0.0.0", port=settings.port)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
111
observatory_read_adapter/app.py
Normal file
111
observatory_read_adapter/app.py
Normal file
|
|
@ -0,0 +1,111 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import hmac
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from aiohttp import web
|
||||||
|
|
||||||
|
from .config import Settings
|
||||||
|
from .db import CloudSqlClaimReader
|
||||||
|
|
||||||
|
logger = logging.getLogger("teleo.observatory_read_adapter")
|
||||||
|
|
||||||
|
SETTINGS_KEY = web.AppKey("observatory_settings", Settings)
|
||||||
|
READER_KEY = web.AppKey("observatory_reader", object)
|
||||||
|
PUBLIC_PATHS = frozenset({"/healthz", "/readyz"})
|
||||||
|
|
||||||
|
|
||||||
|
def _private_json(payload: dict[str, Any], *, status: int = 200) -> web.Response:
|
||||||
|
response = web.json_response(payload, status=status)
|
||||||
|
response.headers["Cache-Control"] = "private, no-store, max-age=0"
|
||||||
|
response.headers["Pragma"] = "no-cache"
|
||||||
|
response.headers["Vary"] = "X-Api-Key"
|
||||||
|
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
@web.middleware
|
||||||
|
async def api_key_auth(request: web.Request, handler: Any) -> web.StreamResponse:
|
||||||
|
if request.path in PUBLIC_PATHS:
|
||||||
|
return await handler(request)
|
||||||
|
settings = request.app[SETTINGS_KEY]
|
||||||
|
provided = request.headers.get("X-Api-Key", "")
|
||||||
|
try:
|
||||||
|
authenticated = hmac.compare_digest(
|
||||||
|
provided.encode("ascii"),
|
||||||
|
settings.api_key.encode("ascii"),
|
||||||
|
)
|
||||||
|
except UnicodeEncodeError:
|
||||||
|
authenticated = False
|
||||||
|
if not authenticated:
|
||||||
|
return _private_json({"error": "unauthorized"}, status=401)
|
||||||
|
return await handler(request)
|
||||||
|
|
||||||
|
|
||||||
|
async def healthz(_request: web.Request) -> web.Response:
|
||||||
|
return web.json_response({"status": "ok"})
|
||||||
|
|
||||||
|
|
||||||
|
async def readyz(request: web.Request) -> web.Response:
|
||||||
|
reader = request.app[READER_KEY]
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(asyncio.to_thread(reader.check_ready), timeout=12.0)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("readiness check failed (%s)", type(exc).__name__)
|
||||||
|
return web.json_response({"status": "unavailable"}, status=503)
|
||||||
|
return web.json_response({"status": "ready"})
|
||||||
|
|
||||||
|
|
||||||
|
async def get_sample_claim(request: web.Request) -> web.Response:
|
||||||
|
return await _claim_response(request, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def get_claim(request: web.Request) -> web.Response:
|
||||||
|
raw_claim_id = request.match_info["claim_id"]
|
||||||
|
try:
|
||||||
|
claim_id = str(uuid.UUID(raw_claim_id))
|
||||||
|
except ValueError:
|
||||||
|
return _private_json({"error": "invalid_claim_id"}, status=400)
|
||||||
|
return await _claim_response(request, claim_id)
|
||||||
|
|
||||||
|
|
||||||
|
async def _claim_response(request: web.Request, claim_id: str | None) -> web.Response:
|
||||||
|
reader = request.app[READER_KEY]
|
||||||
|
try:
|
||||||
|
payload = await asyncio.wait_for(
|
||||||
|
asyncio.to_thread(reader.read_claim, claim_id),
|
||||||
|
timeout=12.0,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("canonical claim read failed (%s)", type(exc).__name__)
|
||||||
|
return _private_json({"error": "canonical_claim_unavailable"}, status=503)
|
||||||
|
if payload is None:
|
||||||
|
return _private_json({"error": "claim_not_found"}, status=404)
|
||||||
|
return _private_json(payload)
|
||||||
|
|
||||||
|
|
||||||
|
async def _cleanup(app: web.Application) -> None:
|
||||||
|
reader = app[READER_KEY]
|
||||||
|
await asyncio.to_thread(reader.close)
|
||||||
|
|
||||||
|
|
||||||
|
def create_app(
|
||||||
|
settings: Settings | None = None,
|
||||||
|
*,
|
||||||
|
reader: Any | None = None,
|
||||||
|
) -> web.Application:
|
||||||
|
settings = settings or Settings.from_env()
|
||||||
|
settings.validate()
|
||||||
|
reader = reader or CloudSqlClaimReader(settings)
|
||||||
|
app = web.Application(middlewares=[api_key_auth], client_max_size=16 * 1024)
|
||||||
|
app[SETTINGS_KEY] = settings
|
||||||
|
app[READER_KEY] = reader
|
||||||
|
app.router.add_get("/healthz", healthz)
|
||||||
|
app.router.add_get("/readyz", readyz)
|
||||||
|
app.router.add_get("/v1/claims/sample", get_sample_claim)
|
||||||
|
app.router.add_get("/v1/claims/{claim_id}", get_claim)
|
||||||
|
app.on_cleanup.append(_cleanup)
|
||||||
|
return app
|
||||||
61
observatory_read_adapter/config.py
Normal file
61
observatory_read_adapter/config.py
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
EXPECTED_PROJECT = "teleo-501523"
|
||||||
|
EXPECTED_REGION = "europe-west6"
|
||||||
|
EXPECTED_INSTANCE = "teleo-pgvector-standby"
|
||||||
|
EXPECTED_CONNECTION_NAME = f"{EXPECTED_PROJECT}:{EXPECTED_REGION}:{EXPECTED_INSTANCE}"
|
||||||
|
EXPECTED_DATABASE = "teleo_canonical"
|
||||||
|
EXPECTED_AUTHORIZATION_ROLE = "kb_observatory_read"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Settings:
|
||||||
|
project_id: str
|
||||||
|
instance_connection_name: str
|
||||||
|
database: str
|
||||||
|
iam_database_user: str
|
||||||
|
authorization_role: str
|
||||||
|
api_key: str = field(repr=False)
|
||||||
|
service_revision: str = "unknown"
|
||||||
|
port: int = 8080
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls) -> Settings:
|
||||||
|
settings = cls(
|
||||||
|
project_id=os.environ.get("GCP_PROJECT_ID", ""),
|
||||||
|
instance_connection_name=os.environ.get("CLOUD_SQL_INSTANCE", ""),
|
||||||
|
database=os.environ.get("DB_NAME", ""),
|
||||||
|
iam_database_user=os.environ.get("DB_IAM_USER", ""),
|
||||||
|
authorization_role=os.environ.get("DB_AUTHORIZATION_ROLE", ""),
|
||||||
|
api_key=os.environ.get("OBSERVATORY_API_KEY", ""),
|
||||||
|
service_revision=os.environ.get("SERVICE_REVISION", "unknown"),
|
||||||
|
port=int(os.environ.get("PORT", "8080")),
|
||||||
|
)
|
||||||
|
settings.validate()
|
||||||
|
return settings
|
||||||
|
|
||||||
|
def validate(self) -> None:
|
||||||
|
if self.project_id != EXPECTED_PROJECT:
|
||||||
|
raise ValueError(f"GCP_PROJECT_ID must be {EXPECTED_PROJECT}")
|
||||||
|
if self.instance_connection_name != EXPECTED_CONNECTION_NAME:
|
||||||
|
raise ValueError(f"CLOUD_SQL_INSTANCE must be {EXPECTED_CONNECTION_NAME}")
|
||||||
|
if self.database != EXPECTED_DATABASE:
|
||||||
|
raise ValueError(f"DB_NAME must be {EXPECTED_DATABASE}")
|
||||||
|
if self.authorization_role != EXPECTED_AUTHORIZATION_ROLE:
|
||||||
|
raise ValueError(f"DB_AUTHORIZATION_ROLE must be {EXPECTED_AUTHORIZATION_ROLE}")
|
||||||
|
if not self.iam_database_user.endswith(f"@{EXPECTED_PROJECT}.iam"):
|
||||||
|
raise ValueError("DB_IAM_USER must be the scoped Cloud SQL service-account user")
|
||||||
|
if any(character.isspace() for character in self.iam_database_user):
|
||||||
|
raise ValueError("DB_IAM_USER must not contain whitespace")
|
||||||
|
if (
|
||||||
|
len(self.api_key) < 32
|
||||||
|
or len(self.api_key) > 512
|
||||||
|
or not self.api_key.isascii()
|
||||||
|
or any(character.isspace() for character in self.api_key)
|
||||||
|
):
|
||||||
|
raise ValueError("OBSERVATORY_API_KEY must be 32-512 non-whitespace ASCII characters")
|
||||||
|
if not 1 <= self.port <= 65535:
|
||||||
|
raise ValueError("PORT is outside the valid TCP port range")
|
||||||
315
observatory_read_adapter/db.py
Normal file
315
observatory_read_adapter/db.py
Normal file
|
|
@ -0,0 +1,315 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Callable
|
||||||
|
from datetime import date, datetime, timezone
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from .config import Settings
|
||||||
|
|
||||||
|
ConnectionFactory = Callable[[], Any]
|
||||||
|
|
||||||
|
|
||||||
|
class ReaderContractError(RuntimeError):
|
||||||
|
"""The live database session does not match the adapter's read-only contract."""
|
||||||
|
|
||||||
|
|
||||||
|
def _json_value(value: Any) -> Any:
|
||||||
|
if isinstance(value, Decimal):
|
||||||
|
return float(value)
|
||||||
|
if isinstance(value, (date, datetime)):
|
||||||
|
return value.isoformat()
|
||||||
|
if isinstance(value, tuple):
|
||||||
|
return list(value)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _column_name(column: Any) -> str:
|
||||||
|
return str(column.name) if hasattr(column, "name") else str(column[0])
|
||||||
|
|
||||||
|
|
||||||
|
def _row_dict(cursor: Any, row: Any) -> dict[str, Any] | None:
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
names = [_column_name(column) for column in cursor.description]
|
||||||
|
return {name: _json_value(value) for name, value in zip(names, row, strict=True)}
|
||||||
|
|
||||||
|
|
||||||
|
def _rows(cursor: Any) -> list[dict[str, Any]]:
|
||||||
|
names = [_column_name(column) for column in cursor.description]
|
||||||
|
return [
|
||||||
|
{name: _json_value(value) for name, value in zip(names, row, strict=True)}
|
||||||
|
for row in cursor.fetchall()
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class CloudSqlClaimReader:
|
||||||
|
def __init__(self, settings: Settings, *, connection_factory: ConnectionFactory | None = None):
|
||||||
|
self.settings = settings
|
||||||
|
self._connector: Any | None = None
|
||||||
|
if connection_factory is not None:
|
||||||
|
self._connection_factory = connection_factory
|
||||||
|
return
|
||||||
|
|
||||||
|
from google.cloud.sql.connector import Connector, IPTypes
|
||||||
|
|
||||||
|
self._connector = Connector(refresh_strategy="LAZY")
|
||||||
|
|
||||||
|
def connect() -> Any:
|
||||||
|
return self._connector.connect(
|
||||||
|
settings.instance_connection_name,
|
||||||
|
"pg8000",
|
||||||
|
user=settings.iam_database_user,
|
||||||
|
db=settings.database,
|
||||||
|
enable_iam_auth=True,
|
||||||
|
ip_type=IPTypes.PRIVATE,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._connection_factory = connect
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
if self._connector is not None:
|
||||||
|
self._connector.close()
|
||||||
|
|
||||||
|
def check_ready(self) -> dict[str, Any]:
|
||||||
|
connection = self._connection_factory()
|
||||||
|
try:
|
||||||
|
cursor = connection.cursor()
|
||||||
|
self._begin_read_only(cursor)
|
||||||
|
session = self._load_session_contract(cursor)
|
||||||
|
connection.rollback()
|
||||||
|
return {
|
||||||
|
"ready": True,
|
||||||
|
"database": session["database"],
|
||||||
|
"authorization_role": self.settings.authorization_role,
|
||||||
|
"transaction_read_only": True,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
def read_claim(self, claim_id: str | None = None) -> dict[str, Any] | None:
|
||||||
|
connection = self._connection_factory()
|
||||||
|
try:
|
||||||
|
cursor = connection.cursor()
|
||||||
|
self._begin_read_only(cursor)
|
||||||
|
session = self._load_session_contract(cursor)
|
||||||
|
claim = self._load_claim(cursor, claim_id)
|
||||||
|
if claim is None:
|
||||||
|
connection.rollback()
|
||||||
|
return None
|
||||||
|
|
||||||
|
evidence = self._load_evidence(cursor, claim["id"])
|
||||||
|
edges = self._load_edges(cursor, claim["id"])
|
||||||
|
proposal_counts = self._load_proposal_counts(cursor)
|
||||||
|
related_proposals = self._load_related_proposals(cursor, claim["id"])
|
||||||
|
recent_proposals = self._load_recent_proposals(cursor)
|
||||||
|
connection.rollback()
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
generated_at = datetime.now(timezone.utc).isoformat()
|
||||||
|
return {
|
||||||
|
"schema": "livingip.observatory-canonical-claim.v1",
|
||||||
|
"generated_at": generated_at,
|
||||||
|
"read_only": True,
|
||||||
|
"provenance": {
|
||||||
|
"gcp_project": self.settings.project_id,
|
||||||
|
"cloud_sql_instance": self.settings.instance_connection_name,
|
||||||
|
"database": session["database"],
|
||||||
|
"database_principal": session["database_principal"],
|
||||||
|
"authorization_role": self.settings.authorization_role,
|
||||||
|
"transaction_read_only": True,
|
||||||
|
"write_privileges_denied": bool(session["required_writes_denied"]),
|
||||||
|
"service_revision": self.settings.service_revision,
|
||||||
|
},
|
||||||
|
"canonical": {
|
||||||
|
"relation": "public.claims",
|
||||||
|
"claim": claim,
|
||||||
|
"evidence_relation": "public.claim_evidence -> public.sources",
|
||||||
|
"evidence": evidence,
|
||||||
|
"edge_relation": "public.claim_edges",
|
||||||
|
"edges": edges,
|
||||||
|
},
|
||||||
|
"proposal_ledger": {
|
||||||
|
"relation": "kb_stage.kb_proposals",
|
||||||
|
"distinct_from_canonical": True,
|
||||||
|
"status_counts": proposal_counts,
|
||||||
|
"related": related_proposals,
|
||||||
|
"recent": recent_proposals,
|
||||||
|
"semantics": {
|
||||||
|
"pending_review": "staged only; not canonical",
|
||||||
|
"approved": "reviewed intent; not canonical until applied_at is set",
|
||||||
|
"applied": "proposal receipt only; canonical rows remain authoritative",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _begin_read_only(cursor: Any) -> None:
|
||||||
|
cursor.execute("set transaction read only")
|
||||||
|
cursor.execute("set local statement_timeout = '5000ms'")
|
||||||
|
cursor.execute("set local lock_timeout = '1000ms'")
|
||||||
|
|
||||||
|
def _load_session_contract(self, cursor: Any) -> dict[str, Any]:
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
select current_database() as database,
|
||||||
|
current_user as database_principal,
|
||||||
|
current_setting('transaction_read_only') as transaction_read_only,
|
||||||
|
pg_has_role(current_user, %s, 'member') as has_authorization_role,
|
||||||
|
has_table_privilege(current_user, 'public.claims', 'select')
|
||||||
|
and has_table_privilege(current_user, 'public.sources', 'select')
|
||||||
|
and has_table_privilege(current_user, 'public.claim_evidence', 'select')
|
||||||
|
and has_table_privilege(current_user, 'public.claim_edges', 'select')
|
||||||
|
and has_table_privilege(current_user, 'kb_stage.kb_proposals', 'select')
|
||||||
|
as has_required_reads,
|
||||||
|
not (
|
||||||
|
has_table_privilege(current_user, 'public.claims', 'insert,update,delete,truncate')
|
||||||
|
or has_table_privilege(current_user, 'public.sources', 'insert,update,delete,truncate')
|
||||||
|
or has_table_privilege(current_user, 'public.claim_evidence', 'insert,update,delete,truncate')
|
||||||
|
or has_table_privilege(current_user, 'public.claim_edges', 'insert,update,delete,truncate')
|
||||||
|
or has_table_privilege(current_user, 'kb_stage.kb_proposals', 'insert,update,delete,truncate')
|
||||||
|
) as required_writes_denied
|
||||||
|
""",
|
||||||
|
(self.settings.authorization_role,),
|
||||||
|
)
|
||||||
|
session = _row_dict(cursor, cursor.fetchone())
|
||||||
|
if session is None:
|
||||||
|
raise ReaderContractError("database session contract returned no row")
|
||||||
|
if session["database"] != self.settings.database:
|
||||||
|
raise ReaderContractError("database session selected an unexpected database")
|
||||||
|
if session["database_principal"] != self.settings.iam_database_user:
|
||||||
|
raise ReaderContractError("database session selected an unexpected principal")
|
||||||
|
if session["transaction_read_only"] != "on":
|
||||||
|
raise ReaderContractError("database session is not read-only")
|
||||||
|
if not session["has_authorization_role"] or not session["has_required_reads"]:
|
||||||
|
raise ReaderContractError("database principal lacks the exact read authorization")
|
||||||
|
if not session["required_writes_denied"]:
|
||||||
|
raise ReaderContractError("database principal has an unexpected write privilege")
|
||||||
|
return session
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _load_claim(cursor: Any, claim_id: str | None) -> dict[str, Any] | None:
|
||||||
|
columns = """
|
||||||
|
select c.id::text as id,
|
||||||
|
c.type::text as type,
|
||||||
|
left(c.text, 12000) as text,
|
||||||
|
c.status::text as status,
|
||||||
|
c.confidence,
|
||||||
|
coalesce(c.tags, '{}'::text[]) as tags,
|
||||||
|
c.superseded_by::text as superseded_by,
|
||||||
|
c.created_at,
|
||||||
|
c.updated_at
|
||||||
|
from public.claims c
|
||||||
|
"""
|
||||||
|
if claim_id is not None:
|
||||||
|
cursor.execute(columns + " where c.id = %s::uuid", (claim_id,))
|
||||||
|
return _row_dict(cursor, cursor.fetchone())
|
||||||
|
|
||||||
|
cursor.execute(
|
||||||
|
columns
|
||||||
|
+ """
|
||||||
|
where exists (
|
||||||
|
select 1 from public.claim_evidence ce where ce.claim_id = c.id
|
||||||
|
)
|
||||||
|
order by coalesce(c.updated_at, c.created_at) desc, c.id desc
|
||||||
|
limit 1
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
claim = _row_dict(cursor, cursor.fetchone())
|
||||||
|
if claim is not None:
|
||||||
|
return claim
|
||||||
|
cursor.execute(columns + " order by coalesce(c.updated_at, c.created_at) desc, c.id desc limit 1")
|
||||||
|
return _row_dict(cursor, cursor.fetchone())
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _load_evidence(cursor: Any, claim_id: str) -> list[dict[str, Any]]:
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
select ce.id::text as evidence_id,
|
||||||
|
ce.role::text as role,
|
||||||
|
ce.weight,
|
||||||
|
ce.created_at as linked_at,
|
||||||
|
s.id::text as source_id,
|
||||||
|
s.source_type::text as source_type,
|
||||||
|
s.url,
|
||||||
|
s.storage_path,
|
||||||
|
left(coalesce(s.excerpt, ''), 2400) as excerpt,
|
||||||
|
s.hash as content_hash,
|
||||||
|
s.captured_at,
|
||||||
|
s.created_at as source_created_at
|
||||||
|
from public.claim_evidence ce
|
||||||
|
join public.sources s on s.id = ce.source_id
|
||||||
|
where ce.claim_id = %s::uuid
|
||||||
|
order by ce.weight desc nulls last, ce.created_at, ce.id
|
||||||
|
limit 30
|
||||||
|
""",
|
||||||
|
(claim_id,),
|
||||||
|
)
|
||||||
|
return _rows(cursor)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _load_edges(cursor: Any, claim_id: str) -> list[dict[str, Any]]:
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
select e.id::text as edge_id,
|
||||||
|
case when e.from_claim = %s::uuid then 'outgoing' else 'incoming' end as direction,
|
||||||
|
e.edge_type::text as edge_type,
|
||||||
|
e.weight,
|
||||||
|
case when e.from_claim = %s::uuid then e.to_claim::text else e.from_claim::text end
|
||||||
|
as connected_claim_id,
|
||||||
|
e.created_at
|
||||||
|
from public.claim_edges e
|
||||||
|
where e.from_claim = %s::uuid or e.to_claim = %s::uuid
|
||||||
|
order by e.created_at, e.id
|
||||||
|
limit 40
|
||||||
|
""",
|
||||||
|
(claim_id, claim_id, claim_id, claim_id),
|
||||||
|
)
|
||||||
|
return _rows(cursor)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _load_proposal_counts(cursor: Any) -> dict[str, int]:
|
||||||
|
cursor.execute(
|
||||||
|
"select status::text as status, count(*)::int as count "
|
||||||
|
"from kb_stage.kb_proposals group by status order by status"
|
||||||
|
)
|
||||||
|
return {str(status): int(count) for status, count in cursor.fetchall()}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _load_related_proposals(cursor: Any, claim_id: str) -> list[dict[str, Any]]:
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
select p.id::text as id,
|
||||||
|
p.proposal_type::text as proposal_type,
|
||||||
|
p.status::text as status,
|
||||||
|
to_jsonb(p)->>'source_ref' as source_ref,
|
||||||
|
p.reviewed_at,
|
||||||
|
p.applied_at,
|
||||||
|
p.updated_at
|
||||||
|
from kb_stage.kb_proposals p
|
||||||
|
where p.payload::text like %s
|
||||||
|
order by p.updated_at desc, p.id desc
|
||||||
|
limit 20
|
||||||
|
""",
|
||||||
|
(f"%{claim_id}%",),
|
||||||
|
)
|
||||||
|
return _rows(cursor)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _load_recent_proposals(cursor: Any) -> list[dict[str, Any]]:
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
select p.id::text as id,
|
||||||
|
p.proposal_type::text as proposal_type,
|
||||||
|
p.status::text as status,
|
||||||
|
to_jsonb(p)->>'source_ref' as source_ref,
|
||||||
|
p.reviewed_at,
|
||||||
|
p.applied_at,
|
||||||
|
p.updated_at
|
||||||
|
from kb_stage.kb_proposals p
|
||||||
|
order by p.updated_at desc, p.id desc
|
||||||
|
limit 10
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
return _rows(cursor)
|
||||||
183
ops/observatory_read_role.sql
Normal file
183
ops/observatory_read_role.sql
Normal file
|
|
@ -0,0 +1,183 @@
|
||||||
|
\set ON_ERROR_STOP on
|
||||||
|
\getenv observatory_iam_user OBSERVATORY_DB_IAM_USER
|
||||||
|
|
||||||
|
-- Provision only after Cloud SQL has created the dedicated IAM database user.
|
||||||
|
-- The service-account username omits the .gserviceaccount.com suffix.
|
||||||
|
select set_config('observatory.iam_user', :'observatory_iam_user', false);
|
||||||
|
|
||||||
|
do $preflight$
|
||||||
|
declare
|
||||||
|
principal text := current_setting('observatory.iam_user');
|
||||||
|
begin
|
||||||
|
if current_database() <> 'teleo_canonical' then
|
||||||
|
raise exception 'observatory_read_role must run only in teleo_canonical';
|
||||||
|
end if;
|
||||||
|
if principal !~ '^[a-z0-9-]+@teleo-501523[.]iam$' then
|
||||||
|
raise exception 'OBSERVATORY_DB_IAM_USER is not the scoped GCP service-account database user';
|
||||||
|
end if;
|
||||||
|
if not exists (select 1 from pg_catalog.pg_roles where rolname = principal) then
|
||||||
|
raise exception 'Cloud SQL IAM database user does not exist: %', principal;
|
||||||
|
end if;
|
||||||
|
end
|
||||||
|
$preflight$;
|
||||||
|
|
||||||
|
select 'create role kb_observatory_read nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1'
|
||||||
|
where not exists (select 1 from pg_catalog.pg_roles where rolname = 'kb_observatory_read')
|
||||||
|
\gexec
|
||||||
|
|
||||||
|
alter role kb_observatory_read
|
||||||
|
with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1
|
||||||
|
password null;
|
||||||
|
alter role kb_observatory_read reset all;
|
||||||
|
|
||||||
|
revoke all privileges on database teleo_canonical from kb_observatory_read;
|
||||||
|
grant connect on database teleo_canonical to kb_observatory_read;
|
||||||
|
|
||||||
|
revoke all on schema public, kb_stage from kb_observatory_read;
|
||||||
|
revoke all privileges on all tables in schema public, kb_stage from kb_observatory_read;
|
||||||
|
revoke all privileges on all sequences in schema public, kb_stage from kb_observatory_read;
|
||||||
|
revoke all privileges on all functions in schema public, kb_stage from kb_observatory_read;
|
||||||
|
revoke all privileges on all procedures in schema public, kb_stage from kb_observatory_read;
|
||||||
|
|
||||||
|
select format(
|
||||||
|
'revoke all privileges (%s) on table %I.%I from kb_observatory_read',
|
||||||
|
pg_catalog.string_agg(pg_catalog.quote_ident(attribute.attname), ', ' order by attribute.attnum),
|
||||||
|
namespace.nspname,
|
||||||
|
relation.relname
|
||||||
|
)
|
||||||
|
from pg_catalog.pg_class relation
|
||||||
|
join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace
|
||||||
|
join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid
|
||||||
|
where namespace.nspname in ('public', 'kb_stage')
|
||||||
|
and relation.relkind in ('r', 'p', 'v', 'm', 'f')
|
||||||
|
and attribute.attnum > 0
|
||||||
|
and not attribute.attisdropped
|
||||||
|
group by namespace.nspname, relation.relname
|
||||||
|
\gexec
|
||||||
|
|
||||||
|
grant usage on schema public, kb_stage to kb_observatory_read;
|
||||||
|
grant select on
|
||||||
|
public.claims,
|
||||||
|
public.sources,
|
||||||
|
public.claim_evidence,
|
||||||
|
public.claim_edges,
|
||||||
|
kb_stage.kb_proposals
|
||||||
|
to kb_observatory_read;
|
||||||
|
|
||||||
|
do $membership$
|
||||||
|
declare
|
||||||
|
principal text := current_setting('observatory.iam_user');
|
||||||
|
unexpected text;
|
||||||
|
begin
|
||||||
|
select pg_catalog.string_agg(granted.rolname, ', ' order by granted.rolname)
|
||||||
|
into unexpected
|
||||||
|
from pg_catalog.pg_auth_members membership
|
||||||
|
join pg_catalog.pg_roles granted on granted.oid = membership.roleid
|
||||||
|
join pg_catalog.pg_roles member on member.oid = membership.member
|
||||||
|
where member.rolname = principal
|
||||||
|
and granted.rolname <> 'kb_observatory_read';
|
||||||
|
if unexpected is not null then
|
||||||
|
raise exception 'dedicated Observatory principal has unexpected role memberships: %', unexpected;
|
||||||
|
end if;
|
||||||
|
|
||||||
|
select pg_catalog.string_agg(member.rolname, ', ' order by member.rolname)
|
||||||
|
into unexpected
|
||||||
|
from pg_catalog.pg_auth_members membership
|
||||||
|
join pg_catalog.pg_roles granted on granted.oid = membership.roleid
|
||||||
|
join pg_catalog.pg_roles member on member.oid = membership.member
|
||||||
|
where granted.rolname = 'kb_observatory_read'
|
||||||
|
and member.rolname <> principal;
|
||||||
|
if unexpected is not null then
|
||||||
|
raise exception 'kb_observatory_read has unexpected members: %', unexpected;
|
||||||
|
end if;
|
||||||
|
|
||||||
|
execute pg_catalog.format('grant kb_observatory_read to %I', principal);
|
||||||
|
execute pg_catalog.format(
|
||||||
|
'alter role %I in database teleo_canonical set default_transaction_read_only = on',
|
||||||
|
principal
|
||||||
|
);
|
||||||
|
execute pg_catalog.format(
|
||||||
|
'alter role %I in database teleo_canonical set statement_timeout = %L',
|
||||||
|
principal,
|
||||||
|
'5s'
|
||||||
|
);
|
||||||
|
execute pg_catalog.format(
|
||||||
|
'alter role %I in database teleo_canonical set lock_timeout = %L',
|
||||||
|
principal,
|
||||||
|
'1s'
|
||||||
|
);
|
||||||
|
end
|
||||||
|
$membership$;
|
||||||
|
|
||||||
|
do $verify$
|
||||||
|
declare
|
||||||
|
principal text := current_setting('observatory.iam_user');
|
||||||
|
unexpected text;
|
||||||
|
begin
|
||||||
|
if not pg_catalog.pg_has_role(principal, 'kb_observatory_read', 'member') then
|
||||||
|
raise exception 'Observatory principal is not a member of kb_observatory_read';
|
||||||
|
end if;
|
||||||
|
if not exists (
|
||||||
|
select 1 from pg_catalog.pg_roles
|
||||||
|
where rolname = principal and rolinherit and not rolsuper and not rolcreatedb
|
||||||
|
and not rolcreaterole and not rolreplication and not rolbypassrls
|
||||||
|
) then
|
||||||
|
raise exception 'Observatory principal has unsafe role attributes or cannot inherit read grants';
|
||||||
|
end if;
|
||||||
|
|
||||||
|
select pg_catalog.string_agg(
|
||||||
|
pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name),
|
||||||
|
', ' order by namespace.nspname, relation.relname, privilege.name
|
||||||
|
)
|
||||||
|
into unexpected
|
||||||
|
from pg_catalog.pg_class relation
|
||||||
|
join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace
|
||||||
|
cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name)
|
||||||
|
where namespace.nspname in ('public', 'kb_stage')
|
||||||
|
and relation.relkind in ('r', 'p', 'v', 'm', 'f')
|
||||||
|
and pg_catalog.has_table_privilege(principal, relation.oid, privilege.name);
|
||||||
|
if unexpected is not null then
|
||||||
|
raise exception 'Observatory principal has effective table writes: %', unexpected;
|
||||||
|
end if;
|
||||||
|
|
||||||
|
select pg_catalog.string_agg(
|
||||||
|
pg_catalog.format('%I.%I', namespace.nspname, relation.relname),
|
||||||
|
', ' order by namespace.nspname, relation.relname
|
||||||
|
)
|
||||||
|
into unexpected
|
||||||
|
from pg_catalog.pg_class relation
|
||||||
|
join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace
|
||||||
|
where namespace.nspname in ('public', 'kb_stage')
|
||||||
|
and relation.relkind in ('r', 'p', 'v', 'm', 'f')
|
||||||
|
and pg_catalog.has_table_privilege(principal, relation.oid, 'SELECT')
|
||||||
|
and (namespace.nspname, relation.relname) not in (
|
||||||
|
('public', 'claims'),
|
||||||
|
('public', 'sources'),
|
||||||
|
('public', 'claim_evidence'),
|
||||||
|
('public', 'claim_edges'),
|
||||||
|
('kb_stage', 'kb_proposals')
|
||||||
|
);
|
||||||
|
if unexpected is not null then
|
||||||
|
raise exception 'Observatory principal can SELECT outside its allowlist: %', unexpected;
|
||||||
|
end if;
|
||||||
|
|
||||||
|
if not (
|
||||||
|
pg_catalog.has_table_privilege(principal, 'public.claims', 'SELECT')
|
||||||
|
and pg_catalog.has_table_privilege(principal, 'public.sources', 'SELECT')
|
||||||
|
and pg_catalog.has_table_privilege(principal, 'public.claim_evidence', 'SELECT')
|
||||||
|
and pg_catalog.has_table_privilege(principal, 'public.claim_edges', 'SELECT')
|
||||||
|
and pg_catalog.has_table_privilege(principal, 'kb_stage.kb_proposals', 'SELECT')
|
||||||
|
) then
|
||||||
|
raise exception 'Observatory principal is missing a required read grant';
|
||||||
|
end if;
|
||||||
|
end
|
||||||
|
$verify$;
|
||||||
|
|
||||||
|
select jsonb_build_object(
|
||||||
|
'database', current_database(),
|
||||||
|
'authorization_role', 'kb_observatory_read',
|
||||||
|
'database_principal', current_setting('observatory.iam_user'),
|
||||||
|
'required_reads', true,
|
||||||
|
'effective_table_writes_denied', true,
|
||||||
|
'default_transaction_read_only', true
|
||||||
|
)::text;
|
||||||
|
|
@ -18,9 +18,12 @@ dev = [
|
||||||
"pytest-asyncio>=0.23",
|
"pytest-asyncio>=0.23",
|
||||||
"ruff>=0.3",
|
"ruff>=0.3",
|
||||||
]
|
]
|
||||||
|
observatory = [
|
||||||
|
"cloud-sql-python-connector[pg8000]>=1.20.4,<1.21",
|
||||||
|
]
|
||||||
|
|
||||||
[tool.setuptools]
|
[tool.setuptools]
|
||||||
packages = ["lib"]
|
packages = ["lib", "observatory_read_adapter"]
|
||||||
|
|
||||||
[tool.ruff]
|
[tool.ruff]
|
||||||
target-version = "py311"
|
target-version = "py311"
|
||||||
|
|
|
||||||
346
tests/test_observatory_read_adapter.py
Normal file
346
tests/test_observatory_read_adapter.py
Normal file
|
|
@ -0,0 +1,346 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from aiohttp.test_utils import TestClient, TestServer
|
||||||
|
|
||||||
|
from observatory_read_adapter.app import create_app
|
||||||
|
from observatory_read_adapter.config import EXPECTED_CONNECTION_NAME, Settings
|
||||||
|
from observatory_read_adapter.db import CloudSqlClaimReader
|
||||||
|
|
||||||
|
CLAIM_ID = "d0000000-0000-0000-0000-000000000005"
|
||||||
|
API_KEY = "observatory-test-api-key-000000000000000000000000"
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def settings(**overrides: object) -> Settings:
|
||||||
|
values = {
|
||||||
|
"project_id": "teleo-501523",
|
||||||
|
"instance_connection_name": EXPECTED_CONNECTION_NAME,
|
||||||
|
"database": "teleo_canonical",
|
||||||
|
"iam_database_user": "sa-observatory-read-adapter@teleo-501523.iam",
|
||||||
|
"authorization_role": "kb_observatory_read",
|
||||||
|
"api_key": API_KEY,
|
||||||
|
"service_revision": "test-revision",
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return Settings(**values)
|
||||||
|
|
||||||
|
|
||||||
|
class FakeReader:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.closed = False
|
||||||
|
|
||||||
|
def check_ready(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"ready": True,
|
||||||
|
"database": "teleo_canonical",
|
||||||
|
"authorization_role": "kb_observatory_read",
|
||||||
|
"transaction_read_only": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
def read_claim(self, claim_id: str | None = None) -> dict[str, object] | None:
|
||||||
|
if claim_id not in (None, CLAIM_ID):
|
||||||
|
return None
|
||||||
|
return {
|
||||||
|
"schema": "livingip.observatory-canonical-claim.v1",
|
||||||
|
"read_only": True,
|
||||||
|
"provenance": {
|
||||||
|
"database": "teleo_canonical",
|
||||||
|
"database_principal": "sa-observatory-read-adapter@teleo-501523.iam",
|
||||||
|
"authorization_role": "kb_observatory_read",
|
||||||
|
"transaction_read_only": True,
|
||||||
|
"write_privileges_denied": True,
|
||||||
|
},
|
||||||
|
"canonical": {
|
||||||
|
"claim": {"id": CLAIM_ID, "status": "open", "text": "Canonical claim"},
|
||||||
|
"evidence": [{"source_id": "e0000000-0000-0000-0000-000000000006"}],
|
||||||
|
},
|
||||||
|
"proposal_ledger": {
|
||||||
|
"distinct_from_canonical": True,
|
||||||
|
"status_counts": {"pending_review": 2, "approved": 1, "applied": 1},
|
||||||
|
"related": [{"status": "approved", "applied_at": None}],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.closed = True
|
||||||
|
|
||||||
|
|
||||||
|
class FakeCursor:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.description: list[tuple[str]] = []
|
||||||
|
self.result: list[tuple[object, ...]] = []
|
||||||
|
self.executed: list[tuple[str, tuple[object, ...]]] = []
|
||||||
|
|
||||||
|
def execute(self, sql: str, parameters: tuple[object, ...] = ()) -> None:
|
||||||
|
normalized = " ".join(sql.split()).lower()
|
||||||
|
self.executed.append((normalized, parameters))
|
||||||
|
self.description = []
|
||||||
|
self.result = []
|
||||||
|
if "current_database() as database" in normalized:
|
||||||
|
self.description = [
|
||||||
|
("database",),
|
||||||
|
("database_principal",),
|
||||||
|
("transaction_read_only",),
|
||||||
|
("has_authorization_role",),
|
||||||
|
("has_required_reads",),
|
||||||
|
("required_writes_denied",),
|
||||||
|
]
|
||||||
|
self.result = [
|
||||||
|
(
|
||||||
|
"teleo_canonical",
|
||||||
|
"sa-observatory-read-adapter@teleo-501523.iam",
|
||||||
|
"on",
|
||||||
|
True,
|
||||||
|
True,
|
||||||
|
True,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
elif "from public.claims c" in normalized:
|
||||||
|
self.description = [
|
||||||
|
("id",),
|
||||||
|
("type",),
|
||||||
|
("text",),
|
||||||
|
("status",),
|
||||||
|
("confidence",),
|
||||||
|
("tags",),
|
||||||
|
("superseded_by",),
|
||||||
|
("created_at",),
|
||||||
|
("updated_at",),
|
||||||
|
]
|
||||||
|
self.result = [
|
||||||
|
(
|
||||||
|
CLAIM_ID,
|
||||||
|
"strategy",
|
||||||
|
"Canonical claim",
|
||||||
|
"open",
|
||||||
|
0.8,
|
||||||
|
["test"],
|
||||||
|
None,
|
||||||
|
"2026-07-15T00:00:00+00:00",
|
||||||
|
"2026-07-15T00:00:00+00:00",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
elif "from public.claim_evidence ce" in normalized:
|
||||||
|
self.description = [
|
||||||
|
("evidence_id",),
|
||||||
|
("role",),
|
||||||
|
("weight",),
|
||||||
|
("linked_at",),
|
||||||
|
("source_id",),
|
||||||
|
("source_type",),
|
||||||
|
("url",),
|
||||||
|
("storage_path",),
|
||||||
|
("excerpt",),
|
||||||
|
("content_hash",),
|
||||||
|
("captured_at",),
|
||||||
|
("source_created_at",),
|
||||||
|
]
|
||||||
|
self.result = [
|
||||||
|
(
|
||||||
|
"e0000000-0000-0000-0000-000000000006",
|
||||||
|
"grounds",
|
||||||
|
1.0,
|
||||||
|
"2026-07-15T00:00:00+00:00",
|
||||||
|
"f0000000-0000-0000-0000-000000000007",
|
||||||
|
"document",
|
||||||
|
"https://example.test/source",
|
||||||
|
None,
|
||||||
|
"Source excerpt",
|
||||||
|
"sha256:test",
|
||||||
|
"2026-07-15T00:00:00+00:00",
|
||||||
|
"2026-07-15T00:00:00+00:00",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
elif "from public.claim_edges e" in normalized:
|
||||||
|
self.description = [
|
||||||
|
("edge_id",),
|
||||||
|
("direction",),
|
||||||
|
("edge_type",),
|
||||||
|
("weight",),
|
||||||
|
("connected_claim_id",),
|
||||||
|
("created_at",),
|
||||||
|
]
|
||||||
|
self.result = []
|
||||||
|
elif "group by status" in normalized:
|
||||||
|
self.description = [("status",), ("count",)]
|
||||||
|
self.result = [("approved", 1), ("applied", 1), ("pending_review", 2)]
|
||||||
|
elif "where p.payload::text like" in normalized:
|
||||||
|
self.description = [
|
||||||
|
("id",),
|
||||||
|
("proposal_type",),
|
||||||
|
("status",),
|
||||||
|
("source_ref",),
|
||||||
|
("reviewed_at",),
|
||||||
|
("applied_at",),
|
||||||
|
("updated_at",),
|
||||||
|
]
|
||||||
|
self.result = [
|
||||||
|
(
|
||||||
|
"a0000000-0000-0000-0000-000000000001",
|
||||||
|
"revise_claim",
|
||||||
|
"approved",
|
||||||
|
"source:test",
|
||||||
|
"2026-07-15T00:00:00+00:00",
|
||||||
|
None,
|
||||||
|
"2026-07-15T00:00:00+00:00",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
elif "from kb_stage.kb_proposals p" in normalized:
|
||||||
|
self.description = [
|
||||||
|
("id",),
|
||||||
|
("proposal_type",),
|
||||||
|
("status",),
|
||||||
|
("source_ref",),
|
||||||
|
("reviewed_at",),
|
||||||
|
("applied_at",),
|
||||||
|
("updated_at",),
|
||||||
|
]
|
||||||
|
self.result = []
|
||||||
|
|
||||||
|
def fetchone(self) -> tuple[object, ...] | None:
|
||||||
|
return self.result.pop(0) if self.result else None
|
||||||
|
|
||||||
|
def fetchall(self) -> list[tuple[object, ...]]:
|
||||||
|
result, self.result = self.result, []
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class FakeConnection:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.cursor_value = FakeCursor()
|
||||||
|
self.rolled_back = False
|
||||||
|
self.closed = False
|
||||||
|
|
||||||
|
def cursor(self) -> FakeCursor:
|
||||||
|
return self.cursor_value
|
||||||
|
|
||||||
|
def rollback(self) -> None:
|
||||||
|
self.rolled_back = True
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.closed = True
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_fail_closed_on_wrong_database_role_or_short_key() -> None:
|
||||||
|
for overrides in (
|
||||||
|
{"database": "teleo"},
|
||||||
|
{"authorization_role": "leoclean_kb_runtime"},
|
||||||
|
{"api_key": "too-short"},
|
||||||
|
{"instance_connection_name": "other:region:instance"},
|
||||||
|
):
|
||||||
|
candidate = settings(**overrides)
|
||||||
|
try:
|
||||||
|
candidate.validate()
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise AssertionError(f"unsafe settings accepted: {overrides}")
|
||||||
|
|
||||||
|
|
||||||
|
def test_authenticated_claim_response_and_negative_http_paths() -> None:
|
||||||
|
async def exercise() -> None:
|
||||||
|
reader = FakeReader()
|
||||||
|
client = TestClient(TestServer(create_app(settings(), reader=reader)))
|
||||||
|
await client.start_server()
|
||||||
|
try:
|
||||||
|
anonymous = await client.get("/v1/claims/sample")
|
||||||
|
assert anonymous.status == 401
|
||||||
|
assert await anonymous.json() == {"error": "unauthorized"}
|
||||||
|
|
||||||
|
response = await client.get(
|
||||||
|
f"/v1/claims/{CLAIM_ID}",
|
||||||
|
headers={"X-Api-Key": API_KEY},
|
||||||
|
)
|
||||||
|
assert response.status == 200
|
||||||
|
assert response.headers["Cache-Control"] == "private, no-store, max-age=0"
|
||||||
|
payload = await response.json()
|
||||||
|
assert payload["schema"] == "livingip.observatory-canonical-claim.v1"
|
||||||
|
assert payload["provenance"]["database"] == "teleo_canonical"
|
||||||
|
assert payload["provenance"]["write_privileges_denied"] is True
|
||||||
|
assert payload["canonical"]["evidence"]
|
||||||
|
assert payload["proposal_ledger"]["distinct_from_canonical"] is True
|
||||||
|
assert payload["proposal_ledger"]["related"][0]["applied_at"] is None
|
||||||
|
assert API_KEY not in json.dumps(payload)
|
||||||
|
|
||||||
|
write_attempt = await client.post(
|
||||||
|
f"/v1/claims/{CLAIM_ID}",
|
||||||
|
headers={"X-Api-Key": API_KEY},
|
||||||
|
json={"status": "applied"},
|
||||||
|
)
|
||||||
|
assert write_attempt.status == 405
|
||||||
|
|
||||||
|
invalid = await client.get(
|
||||||
|
"/v1/claims/not-a-uuid",
|
||||||
|
headers={"X-Api-Key": API_KEY},
|
||||||
|
)
|
||||||
|
assert invalid.status == 400
|
||||||
|
finally:
|
||||||
|
await client.close()
|
||||||
|
assert reader.closed is True
|
||||||
|
|
||||||
|
asyncio.run(exercise())
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_checks_database_contract_without_exposing_claims() -> None:
|
||||||
|
async def exercise() -> None:
|
||||||
|
client = TestClient(TestServer(create_app(settings(), reader=FakeReader())))
|
||||||
|
await client.start_server()
|
||||||
|
try:
|
||||||
|
response = await client.get("/readyz")
|
||||||
|
payload = await response.json()
|
||||||
|
assert response.status == 200
|
||||||
|
assert payload == {"status": "ready"}
|
||||||
|
finally:
|
||||||
|
await client.close()
|
||||||
|
|
||||||
|
asyncio.run(exercise())
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_reader_returns_provenance_and_keeps_proposals_distinct() -> None:
|
||||||
|
connection = FakeConnection()
|
||||||
|
reader = CloudSqlClaimReader(settings(), connection_factory=lambda: connection)
|
||||||
|
|
||||||
|
payload = reader.read_claim(CLAIM_ID)
|
||||||
|
|
||||||
|
assert payload is not None
|
||||||
|
assert payload["provenance"]["database"] == "teleo_canonical"
|
||||||
|
assert payload["provenance"]["database_principal"] == settings().iam_database_user
|
||||||
|
assert payload["provenance"]["write_privileges_denied"] is True
|
||||||
|
assert payload["canonical"]["claim"]["id"] == CLAIM_ID
|
||||||
|
assert payload["canonical"]["evidence"][0]["content_hash"] == "sha256:test"
|
||||||
|
assert payload["proposal_ledger"]["distinct_from_canonical"] is True
|
||||||
|
assert payload["proposal_ledger"]["related"][0]["status"] == "approved"
|
||||||
|
assert payload["proposal_ledger"]["related"][0]["applied_at"] is None
|
||||||
|
assert connection.rolled_back is True
|
||||||
|
assert connection.closed is True
|
||||||
|
assert any("set transaction read only" in sql for sql, _parameters in connection.cursor_value.executed)
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_role_contract_is_select_only_and_iam_scoped() -> None:
|
||||||
|
sql = (ROOT / "ops" / "observatory_read_role.sql").read_text(encoding="utf-8")
|
||||||
|
lowered = sql.lower()
|
||||||
|
|
||||||
|
assert "kb_observatory_read nologin" in lowered
|
||||||
|
assert "default_transaction_read_only = on" in lowered
|
||||||
|
assert "public.claims" in lowered
|
||||||
|
assert "public.sources" in lowered
|
||||||
|
assert "public.claim_evidence" in lowered
|
||||||
|
assert "public.claim_edges" in lowered
|
||||||
|
assert "kb_stage.kb_proposals" in lowered
|
||||||
|
assert "grant select on" in lowered
|
||||||
|
assert "grant insert" not in lowered
|
||||||
|
assert "grant update" not in lowered
|
||||||
|
assert "grant delete" not in lowered
|
||||||
|
assert "effective_table_writes_denied" in lowered
|
||||||
|
|
||||||
|
|
||||||
|
def test_container_runs_as_non_root_and_contains_no_password_contract() -> None:
|
||||||
|
dockerfile = (ROOT / "Dockerfile.observatory-read-adapter").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "USER 10001:10001" in dockerfile
|
||||||
|
assert "PGPASSWORD" not in dockerfile
|
||||||
|
assert "DB_PASS" not in dockerfile
|
||||||
182
tests/test_observatory_read_role_postgres.py
Normal file
182
tests/test_observatory_read_role_postgres.py
Normal file
|
|
@ -0,0 +1,182 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
ROLE_SQL = ROOT / "ops" / "observatory_read_role.sql"
|
||||||
|
POSTGRES_IMAGE = "postgres:16-alpine"
|
||||||
|
DATABASE = "teleo_canonical"
|
||||||
|
IAM_USER = "sa-observatory-read-adapter@teleo-501523.iam"
|
||||||
|
TEST_PASSWORD = "generated-test-only-password"
|
||||||
|
|
||||||
|
BOOTSTRAP_SQL = f"""
|
||||||
|
begin;
|
||||||
|
create schema kb_stage;
|
||||||
|
create role "{IAM_USER}" login inherit password '{TEST_PASSWORD}';
|
||||||
|
create table public.claims (id uuid primary key);
|
||||||
|
create table public.sources (id uuid primary key);
|
||||||
|
create table public.claim_evidence (id uuid primary key);
|
||||||
|
create table public.claim_edges (id uuid primary key);
|
||||||
|
create table public.private_notes (id uuid primary key);
|
||||||
|
create table kb_stage.kb_proposals (id uuid primary key);
|
||||||
|
insert into public.claims values ('11111111-1111-1111-1111-111111111111');
|
||||||
|
commit;
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def run(command: list[str], *, input_text: str | None = None, check: bool = True) -> subprocess.CompletedProcess[str]:
|
||||||
|
return subprocess.run(
|
||||||
|
command,
|
||||||
|
input=input_text,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=check,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker is required")
|
||||||
|
def test_observatory_role_is_read_only_and_exactly_allowlisted() -> None:
|
||||||
|
container = f"teleo-observatory-role-{uuid.uuid4().hex[:10]}"
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--detach",
|
||||||
|
"--rm",
|
||||||
|
"--name",
|
||||||
|
container,
|
||||||
|
"--env",
|
||||||
|
f"POSTGRES_PASSWORD={TEST_PASSWORD}",
|
||||||
|
"--env",
|
||||||
|
f"POSTGRES_DB={DATABASE}",
|
||||||
|
POSTGRES_IMAGE,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
deadline = time.monotonic() + 25
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
ready = run(
|
||||||
|
["docker", "exec", container, "pg_isready", "-U", "postgres", "-d", DATABASE],
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if ready.returncode == 0:
|
||||||
|
break
|
||||||
|
time.sleep(0.25)
|
||||||
|
else:
|
||||||
|
raise AssertionError("ephemeral Postgres did not become ready")
|
||||||
|
|
||||||
|
bootstrap: subprocess.CompletedProcess[str] | None = None
|
||||||
|
deadline = time.monotonic() + 15
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
bootstrap = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
"-i",
|
||||||
|
container,
|
||||||
|
"psql",
|
||||||
|
"--set",
|
||||||
|
"ON_ERROR_STOP=1",
|
||||||
|
"-U",
|
||||||
|
"postgres",
|
||||||
|
"-d",
|
||||||
|
DATABASE,
|
||||||
|
],
|
||||||
|
input_text=BOOTSTRAP_SQL,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if bootstrap.returncode == 0:
|
||||||
|
break
|
||||||
|
time.sleep(0.25)
|
||||||
|
else:
|
||||||
|
assert bootstrap is not None
|
||||||
|
raise AssertionError(f"ephemeral Postgres bootstrap failed: {bootstrap.stderr[-1000:]}")
|
||||||
|
migration = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
"-e",
|
||||||
|
f"OBSERVATORY_DB_IAM_USER={IAM_USER}",
|
||||||
|
"-i",
|
||||||
|
container,
|
||||||
|
"psql",
|
||||||
|
"-U",
|
||||||
|
"postgres",
|
||||||
|
"-d",
|
||||||
|
DATABASE,
|
||||||
|
],
|
||||||
|
input_text=ROLE_SQL.read_text(encoding="utf-8"),
|
||||||
|
)
|
||||||
|
assert '"effective_table_writes_denied": true' in migration.stdout
|
||||||
|
|
||||||
|
readback = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
"-e",
|
||||||
|
f"PGPASSWORD={TEST_PASSWORD}",
|
||||||
|
container,
|
||||||
|
"psql",
|
||||||
|
"-h",
|
||||||
|
"127.0.0.1",
|
||||||
|
"-U",
|
||||||
|
IAM_USER,
|
||||||
|
"-d",
|
||||||
|
DATABASE,
|
||||||
|
"-At",
|
||||||
|
"-c",
|
||||||
|
"show default_transaction_read_only; select count(*) from public.claims;",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
assert readback.stdout.splitlines() == ["on", "1"]
|
||||||
|
|
||||||
|
write_attempt = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
"-e",
|
||||||
|
f"PGPASSWORD={TEST_PASSWORD}",
|
||||||
|
container,
|
||||||
|
"psql",
|
||||||
|
"-h",
|
||||||
|
"127.0.0.1",
|
||||||
|
"-U",
|
||||||
|
IAM_USER,
|
||||||
|
"-d",
|
||||||
|
DATABASE,
|
||||||
|
"-c",
|
||||||
|
"insert into public.claims values ('22222222-2222-2222-2222-222222222222');",
|
||||||
|
],
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert write_attempt.returncode != 0
|
||||||
|
|
||||||
|
outside_allowlist = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
"-e",
|
||||||
|
f"PGPASSWORD={TEST_PASSWORD}",
|
||||||
|
container,
|
||||||
|
"psql",
|
||||||
|
"-h",
|
||||||
|
"127.0.0.1",
|
||||||
|
"-U",
|
||||||
|
IAM_USER,
|
||||||
|
"-d",
|
||||||
|
DATABASE,
|
||||||
|
"-c",
|
||||||
|
"select * from public.private_notes;",
|
||||||
|
],
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert outside_allowlist.returncode != 0
|
||||||
|
finally:
|
||||||
|
run(["docker", "rm", "--force", container], check=False)
|
||||||
Loading…
Reference in a new issue