Add authenticated GCP Observatory read adapter (#154)
Some checks are pending
CI / lint-and-test (push) Waiting to run
Some checks are pending
CI / lint-and-test (push) Waiting to run
This commit is contained in:
parent
15c30f1fbe
commit
92da54d804
13 changed files with 1559 additions and 1 deletions
199
.github/workflows/gcp-observatory-read-adapter.yml
vendored
Normal file
199
.github/workflows/gcp-observatory-read-adapter.yml
vendored
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
name: gcp-observatory-read-adapter
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
action:
|
||||
description: Build only, or deploy the protected staging service and run live receipts
|
||||
required: true
|
||||
default: build_only
|
||||
type: choice
|
||||
options:
|
||||
- build_only
|
||||
- deploy_staging
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: gcp-observatory-read-adapter-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROJECT_ID: teleo-501523
|
||||
REGION: europe-west6
|
||||
ZONE: europe-west6-a
|
||||
REPOSITORY: teleo
|
||||
IMAGE_NAME: observatory-read-adapter
|
||||
SERVICE_NAME: observatory-read-adapter-staging
|
||||
RUNTIME_SERVICE_ACCOUNT: sa-observatory-read-adapter@teleo-501523.iam.gserviceaccount.com
|
||||
DB_IAM_USER: sa-observatory-read-adapter@teleo-501523.iam
|
||||
API_KEY_SECRET: observatory-read-api-key-staging
|
||||
WORKLOAD_IDENTITY_PROVIDER: projects/785938879453/locations/global/workloadIdentityPools/github-actions/providers/living-ip-github
|
||||
DEPLOY_SERVICE_ACCOUNT: sa-artifact-builder@teleo-501523.iam.gserviceaccount.com
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Test, build, and push adapter image
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
outputs:
|
||||
image_ref: ${{ steps.image.outputs.image_ref }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: pip
|
||||
|
||||
- name: Run focused adapter tests
|
||||
run: |
|
||||
python -m pip install -e '.[dev]'
|
||||
python -m pytest tests/test_observatory_read_adapter.py -q
|
||||
python -m ruff check observatory_read_adapter tests/test_observatory_read_adapter.py
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v3
|
||||
with:
|
||||
workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ env.DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v3
|
||||
with:
|
||||
project_id: ${{ env.PROJECT_ID }}
|
||||
|
||||
- name: Configure Artifact Registry
|
||||
run: gcloud auth configure-docker "${REGION}-docker.pkg.dev" --quiet
|
||||
|
||||
- id: image
|
||||
name: Build, smoke, and push immutable image
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag="${GITHUB_SHA::12}"
|
||||
image="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPOSITORY}/${IMAGE_NAME}:${tag}"
|
||||
docker build -f Dockerfile.observatory-read-adapter -t "${image}" .
|
||||
docker run --rm --env PYTHONPYCACHEPREFIX=/tmp/pycache \
|
||||
"${image}" python -m compileall -q /app/observatory_read_adapter
|
||||
docker run --rm "${image}" python -c 'import observatory_read_adapter; print("adapter-import-ok")'
|
||||
docker push "${image}" | tee docker-push.log
|
||||
digest="$(awk '/digest: sha256:/ {print $3}' docker-push.log | tail -1)"
|
||||
test -n "${digest}"
|
||||
image_ref="${image}@${digest}"
|
||||
printf 'image_ref=%s\n' "${image_ref}" >> "${GITHUB_OUTPUT}"
|
||||
printf 'revision=%s\nimage_ref=%s\n' "${GITHUB_SHA}" "${image_ref}" > observatory-adapter-image.txt
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: observatory-adapter-image
|
||||
path: observatory-adapter-image.txt
|
||||
if-no-files-found: error
|
||||
|
||||
deploy-staging:
|
||||
name: Deploy and prove staging read path
|
||||
if: ${{ inputs.action == 'deploy_staging' }}
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v3
|
||||
with:
|
||||
workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ env.DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v3
|
||||
with:
|
||||
project_id: ${{ env.PROJECT_ID }}
|
||||
|
||||
- name: Deploy bounded GCP staging service
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE_REF: ${{ needs.build.outputs.image_ref }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
gcloud run deploy "${SERVICE_NAME}" \
|
||||
--project="${PROJECT_ID}" \
|
||||
--region="${REGION}" \
|
||||
--image="${IMAGE_REF}" \
|
||||
--execution-environment=gen2 \
|
||||
--service-account="${RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--network=teleo-staging-net \
|
||||
--subnet=teleo-staging-europe-west6 \
|
||||
--vpc-egress=private-ranges-only \
|
||||
--ingress=all \
|
||||
--allow-unauthenticated \
|
||||
--port=8080 \
|
||||
--cpu=1 \
|
||||
--memory=512Mi \
|
||||
--concurrency=8 \
|
||||
--max-instances=2 \
|
||||
--timeout=15 \
|
||||
--set-secrets="OBSERVATORY_API_KEY=${API_KEY_SECRET}:latest" \
|
||||
--set-env-vars="GCP_PROJECT_ID=${PROJECT_ID},CLOUD_SQL_INSTANCE=${PROJECT_ID}:${REGION}:teleo-pgvector-standby,DB_NAME=teleo_canonical,DB_IAM_USER=${DB_IAM_USER},DB_AUTHORIZATION_ROLE=kb_observatory_read,SERVICE_REVISION=${GITHUB_SHA}" \
|
||||
--labels="livingip-tier=gcp-staging,livingip-surface=observatory-read-adapter" \
|
||||
--quiet
|
||||
|
||||
- name: Capture positive and negative live receipts
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
service_url="$(gcloud run services describe "${SERVICE_NAME}" --project="${PROJECT_ID}" --region="${REGION}" --format='value(status.url)')"
|
||||
revision="$(gcloud run services describe "${SERVICE_NAME}" --project="${PROJECT_ID}" --region="${REGION}" --format='value(status.latestReadyRevisionName)')"
|
||||
api_key="$(gcloud secrets versions access latest --secret="${API_KEY_SECRET}" --project="${PROJECT_ID}")"
|
||||
test -n "${service_url}"
|
||||
test -n "${revision}"
|
||||
test -n "${api_key}"
|
||||
|
||||
curl --fail --silent --show-error \
|
||||
--header "X-Api-Key: ${api_key}" \
|
||||
"${service_url}/v1/claims/sample" > positive.json
|
||||
jq -e '
|
||||
.schema == "livingip.observatory-canonical-claim.v1"
|
||||
and .read_only == true
|
||||
and .provenance.database == "teleo_canonical"
|
||||
and .provenance.authorization_role == "kb_observatory_read"
|
||||
and .provenance.transaction_read_only == true
|
||||
and .provenance.write_privileges_denied == true
|
||||
and (.canonical.evidence | length) > 0
|
||||
and .proposal_ledger.distinct_from_canonical == true
|
||||
' positive.json >/dev/null
|
||||
|
||||
anonymous_status="$(curl --silent --output anonymous.json --write-out '%{http_code}' "${service_url}/v1/claims/sample")"
|
||||
write_status="$(curl --silent --output write.json --write-out '%{http_code}' \
|
||||
--request POST --header "X-Api-Key: ${api_key}" --header 'Content-Type: application/json' \
|
||||
--data '{"status":"applied"}' "${service_url}/v1/claims/sample")"
|
||||
test "${anonymous_status}" = 401
|
||||
test "${write_status}" = 405
|
||||
|
||||
jq -n \
|
||||
--arg service_url "${service_url}" \
|
||||
--arg revision "${revision}" \
|
||||
--arg git_sha "${GITHUB_SHA}" \
|
||||
--argjson positive "$(cat positive.json)" \
|
||||
--arg anonymous_status "${anonymous_status}" \
|
||||
--arg write_status "${write_status}" \
|
||||
'{
|
||||
schema: "livingip.observatory-read-adapter-live-receipt.v1",
|
||||
required_tier: "T3_live_readonly",
|
||||
service_url: $service_url,
|
||||
revision: $revision,
|
||||
git_sha: $git_sha,
|
||||
positive: $positive,
|
||||
negative: {
|
||||
anonymous_http_status: ($anonymous_status | tonumber),
|
||||
authenticated_post_http_status: ($write_status | tonumber)
|
||||
},
|
||||
production_repoint_executed: false
|
||||
}' > observatory-read-adapter-live-receipt.json
|
||||
unset api_key
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: observatory-read-adapter-live-receipt
|
||||
path: observatory-read-adapter-live-receipt.json
|
||||
if-no-files-found: error
|
||||
20
Dockerfile.observatory-read-adapter
Normal file
20
Dockerfile.observatory-read-adapter
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
FROM python:3.11-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PORT=8080
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY pyproject.toml README.md /app/
|
||||
COPY observatory_read_adapter /app/observatory_read_adapter
|
||||
COPY lib /app/lib
|
||||
|
||||
RUN pip install --no-cache-dir --upgrade pip \
|
||||
&& pip install --no-cache-dir ".[observatory]" \
|
||||
&& addgroup --system --gid 10001 adapter \
|
||||
&& adduser --system --uid 10001 --ingroup adapter --no-create-home adapter
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
CMD ["python", "-m", "observatory_read_adapter"]
|
||||
102
docs/observatory-read-adapter-gcp-staging.md
Normal file
102
docs/observatory-read-adapter-gcp-staging.md
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
# GCP Observatory Read Adapter
|
||||
|
||||
## Definition of Working
|
||||
|
||||
- Working target: an authenticated GCP staging API returns one canonical claim
|
||||
with source/evidence provenance and a separately labeled proposal ledger.
|
||||
- Operator path: `GET /v1/claims/sample` or `GET /v1/claims/{uuid}` with the
|
||||
server-only `X-Api-Key` value.
|
||||
- Done means: the response names `teleo_canonical`, the dedicated IAM database
|
||||
principal and `kb_observatory_read`, reports a read-only transaction and
|
||||
denied write privileges, and live anonymous/write HTTP attempts return
|
||||
`401`/`405`.
|
||||
- Not done: the VPS `teleo-pg`, a public Cloud SQL address, a browser-visible
|
||||
key, a mock-only receipt, or a production Vercel repoint.
|
||||
- Required tier: `T3_live_readonly`.
|
||||
|
||||
## Existing Connector Boundary
|
||||
|
||||
The current `living-ip/livingip-web` main branch reads claims through
|
||||
`src/lib/api.ts`. It selects `NEXT_PUBLIC_API_URL` and otherwise falls back to
|
||||
`http://77.42.65.182:8081`, then calls the legacy `/api/claims` routes. That is
|
||||
the VPS-local/public Observatory path. It is not safe to replace that public
|
||||
environment variable with this protected API: the response shape differs and
|
||||
the adapter key must never enter a `NEXT_PUBLIC_` variable or browser bundle.
|
||||
|
||||
The existing Teleo route `GET /api/kb/claims` is the compatibility reference
|
||||
for route-scoped authentication, but its default database/container settings
|
||||
still describe the VPS-local process. This service is a separate GCP runtime;
|
||||
it does not change Argus or PR #148's Leo runtime files.
|
||||
|
||||
## Runtime Contract
|
||||
|
||||
- Cloud Run service: `observatory-read-adapter-staging` in `europe-west6`.
|
||||
- Direct VPC egress: `teleo-staging-net` / `teleo-staging-europe-west6`.
|
||||
- Cloud SQL: `teleo-501523:europe-west6:teleo-pgvector-standby`, private IP.
|
||||
- Database: `teleo_canonical` only.
|
||||
- Runtime identity: `sa-observatory-read-adapter@teleo-501523.iam.gserviceaccount.com`.
|
||||
- IAM database user: `sa-observatory-read-adapter@teleo-501523.iam`.
|
||||
- Database authorization role: `kb_observatory_read` (`NOLOGIN`).
|
||||
- API authentication: route-scoped key from Secret Manager secret
|
||||
`observatory-read-api-key-staging`; the value is not in Git or deploy logs.
|
||||
- No mutation routes exist. Authenticated unsupported methods return `405`.
|
||||
|
||||
The Cloud SQL Python Connector uses Application Default Credentials from the
|
||||
Cloud Run service identity, automatic IAM database authentication, and private
|
||||
IP. Every request starts a read-only transaction and refuses the response if
|
||||
the database, IAM principal, authorization membership, required reads, or
|
||||
effective write-denial checks drift.
|
||||
|
||||
## One-Time Staging Bootstrap
|
||||
|
||||
These are staging mutations. Run them only with an authenticated GCP operator;
|
||||
they do not route production traffic.
|
||||
|
||||
1. Create the dedicated runtime service account and grant only
|
||||
`roles/cloudsql.client` and `roles/cloudsql.instanceUser` in project
|
||||
`teleo-501523`.
|
||||
2. Add the service account as a Cloud SQL IAM service-account user on
|
||||
`teleo-pgvector-standby`.
|
||||
3. From the existing private GCP VM database-admin path, run
|
||||
`ops/observatory_read_role.sql` against `teleo_canonical` with
|
||||
`OBSERVATORY_DB_IAM_USER=sa-observatory-read-adapter@teleo-501523.iam`.
|
||||
Do not read, copy, or retain the administrator password.
|
||||
4. Create `observatory-read-api-key-staging`, add a generated 32-byte-or-longer
|
||||
value through stdin, and grant that secret's accessor role only to the
|
||||
runtime service account and the bounded staging canary identity.
|
||||
5. Dispatch `.github/workflows/gcp-observatory-read-adapter.yml` with
|
||||
`action=deploy_staging` from the exact reviewed branch revision.
|
||||
|
||||
The workflow builds an immutable image, deploys at most two Cloud Run
|
||||
instances, and retains the positive response plus anonymous `401` and
|
||||
authenticated POST `405` receipts. It does not modify Vercel.
|
||||
|
||||
## Unexecuted Cutover Packet
|
||||
|
||||
Production cutover requires a separate exact authorization and a
|
||||
`living-ip/livingip-web` change. Do not execute these steps from this lane.
|
||||
|
||||
1. Add server-only Vercel secrets `OBSERVATORY_CANONICAL_API_URL` and
|
||||
`OBSERVATORY_CANONICAL_API_KEY` to a protected preview environment. The URL
|
||||
targets the GCP service; neither setting may use `NEXT_PUBLIC_`.
|
||||
2. Add a Next.js server route or server action that calls the GCP API, validates
|
||||
`livingip.observatory-canonical-claim.v1`, and maps it to the Observatory UI.
|
||||
The browser calls only the same-origin Next.js route.
|
||||
3. Prove preview Deployment Protection, anonymous denial, source/evidence
|
||||
rendering, proposal/live labels, and no browser key exposure.
|
||||
4. Obtain exact production-repoint authorization naming the reviewed
|
||||
`livingip-web` commit and Vercel project before changing production values.
|
||||
|
||||
## Unexecuted Rollback Packet
|
||||
|
||||
1. Revert the authorized `livingip-web` connector commit or disable its
|
||||
server-side feature flag.
|
||||
2. Restore the prior production Vercel values and redeploy the last accepted
|
||||
production revision.
|
||||
3. Verify `/claims` again uses the prior `/api/claims` contract.
|
||||
4. Leave the private Cloud SQL instance and adapter role intact while receipts
|
||||
are reviewed; scale the staging Cloud Run service to zero if isolation is
|
||||
required.
|
||||
5. Delete the staging service/role/secret only under a separate cleanup
|
||||
decision. Rollback never adds a Cloud SQL public IP and never redirects the
|
||||
adapter to VPS `teleo-pg`.
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"fallback_goal_path": "goal.md",
|
||||
"platform_goal_after": {
|
||||
"objective": "Complete and verify the private GCP Observatory read adapter defined in this goal file, stopping before production repoint without exact authorization.",
|
||||
"status": "active",
|
||||
"thread_id": "019f6350-69fa-71f3-944b-3fc0065f4fec"
|
||||
},
|
||||
"platform_goal_before": null,
|
||||
"platform_goal_repair_action": "create_goal",
|
||||
"platform_goal_replaced": false,
|
||||
"why_not_replaced": "No prior platform Goal existed; a new active Goal was created from goal.md."
|
||||
}
|
||||
6
observatory_read_adapter/__init__.py
Normal file
6
observatory_read_adapter/__init__.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
"""Authenticated, read-only Observatory adapter for canonical Teleo claims."""
|
||||
|
||||
from .app import create_app
|
||||
from .config import Settings
|
||||
|
||||
__all__ = ["Settings", "create_app"]
|
||||
18
observatory_read_adapter/__main__.py
Normal file
18
observatory_read_adapter/__main__.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .app import create_app
|
||||
from .config import Settings
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
settings = Settings.from_env()
|
||||
web.run_app(create_app(settings), host="0.0.0.0", port=settings.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
111
observatory_read_adapter/app.py
Normal file
111
observatory_read_adapter/app.py
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hmac
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .config import Settings
|
||||
from .db import CloudSqlClaimReader
|
||||
|
||||
logger = logging.getLogger("teleo.observatory_read_adapter")
|
||||
|
||||
SETTINGS_KEY = web.AppKey("observatory_settings", Settings)
|
||||
READER_KEY = web.AppKey("observatory_reader", object)
|
||||
PUBLIC_PATHS = frozenset({"/healthz", "/readyz"})
|
||||
|
||||
|
||||
def _private_json(payload: dict[str, Any], *, status: int = 200) -> web.Response:
|
||||
response = web.json_response(payload, status=status)
|
||||
response.headers["Cache-Control"] = "private, no-store, max-age=0"
|
||||
response.headers["Pragma"] = "no-cache"
|
||||
response.headers["Vary"] = "X-Api-Key"
|
||||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||
return response
|
||||
|
||||
|
||||
@web.middleware
|
||||
async def api_key_auth(request: web.Request, handler: Any) -> web.StreamResponse:
|
||||
if request.path in PUBLIC_PATHS:
|
||||
return await handler(request)
|
||||
settings = request.app[SETTINGS_KEY]
|
||||
provided = request.headers.get("X-Api-Key", "")
|
||||
try:
|
||||
authenticated = hmac.compare_digest(
|
||||
provided.encode("ascii"),
|
||||
settings.api_key.encode("ascii"),
|
||||
)
|
||||
except UnicodeEncodeError:
|
||||
authenticated = False
|
||||
if not authenticated:
|
||||
return _private_json({"error": "unauthorized"}, status=401)
|
||||
return await handler(request)
|
||||
|
||||
|
||||
async def healthz(_request: web.Request) -> web.Response:
|
||||
return web.json_response({"status": "ok"})
|
||||
|
||||
|
||||
async def readyz(request: web.Request) -> web.Response:
|
||||
reader = request.app[READER_KEY]
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.to_thread(reader.check_ready), timeout=12.0)
|
||||
except Exception as exc:
|
||||
logger.warning("readiness check failed (%s)", type(exc).__name__)
|
||||
return web.json_response({"status": "unavailable"}, status=503)
|
||||
return web.json_response({"status": "ready"})
|
||||
|
||||
|
||||
async def get_sample_claim(request: web.Request) -> web.Response:
|
||||
return await _claim_response(request, None)
|
||||
|
||||
|
||||
async def get_claim(request: web.Request) -> web.Response:
|
||||
raw_claim_id = request.match_info["claim_id"]
|
||||
try:
|
||||
claim_id = str(uuid.UUID(raw_claim_id))
|
||||
except ValueError:
|
||||
return _private_json({"error": "invalid_claim_id"}, status=400)
|
||||
return await _claim_response(request, claim_id)
|
||||
|
||||
|
||||
async def _claim_response(request: web.Request, claim_id: str | None) -> web.Response:
|
||||
reader = request.app[READER_KEY]
|
||||
try:
|
||||
payload = await asyncio.wait_for(
|
||||
asyncio.to_thread(reader.read_claim, claim_id),
|
||||
timeout=12.0,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("canonical claim read failed (%s)", type(exc).__name__)
|
||||
return _private_json({"error": "canonical_claim_unavailable"}, status=503)
|
||||
if payload is None:
|
||||
return _private_json({"error": "claim_not_found"}, status=404)
|
||||
return _private_json(payload)
|
||||
|
||||
|
||||
async def _cleanup(app: web.Application) -> None:
|
||||
reader = app[READER_KEY]
|
||||
await asyncio.to_thread(reader.close)
|
||||
|
||||
|
||||
def create_app(
|
||||
settings: Settings | None = None,
|
||||
*,
|
||||
reader: Any | None = None,
|
||||
) -> web.Application:
|
||||
settings = settings or Settings.from_env()
|
||||
settings.validate()
|
||||
reader = reader or CloudSqlClaimReader(settings)
|
||||
app = web.Application(middlewares=[api_key_auth], client_max_size=16 * 1024)
|
||||
app[SETTINGS_KEY] = settings
|
||||
app[READER_KEY] = reader
|
||||
app.router.add_get("/healthz", healthz)
|
||||
app.router.add_get("/readyz", readyz)
|
||||
app.router.add_get("/v1/claims/sample", get_sample_claim)
|
||||
app.router.add_get("/v1/claims/{claim_id}", get_claim)
|
||||
app.on_cleanup.append(_cleanup)
|
||||
return app
|
||||
61
observatory_read_adapter/config.py
Normal file
61
observatory_read_adapter/config.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
EXPECTED_PROJECT = "teleo-501523"
|
||||
EXPECTED_REGION = "europe-west6"
|
||||
EXPECTED_INSTANCE = "teleo-pgvector-standby"
|
||||
EXPECTED_CONNECTION_NAME = f"{EXPECTED_PROJECT}:{EXPECTED_REGION}:{EXPECTED_INSTANCE}"
|
||||
EXPECTED_DATABASE = "teleo_canonical"
|
||||
EXPECTED_AUTHORIZATION_ROLE = "kb_observatory_read"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
project_id: str
|
||||
instance_connection_name: str
|
||||
database: str
|
||||
iam_database_user: str
|
||||
authorization_role: str
|
||||
api_key: str = field(repr=False)
|
||||
service_revision: str = "unknown"
|
||||
port: int = 8080
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> Settings:
|
||||
settings = cls(
|
||||
project_id=os.environ.get("GCP_PROJECT_ID", ""),
|
||||
instance_connection_name=os.environ.get("CLOUD_SQL_INSTANCE", ""),
|
||||
database=os.environ.get("DB_NAME", ""),
|
||||
iam_database_user=os.environ.get("DB_IAM_USER", ""),
|
||||
authorization_role=os.environ.get("DB_AUTHORIZATION_ROLE", ""),
|
||||
api_key=os.environ.get("OBSERVATORY_API_KEY", ""),
|
||||
service_revision=os.environ.get("SERVICE_REVISION", "unknown"),
|
||||
port=int(os.environ.get("PORT", "8080")),
|
||||
)
|
||||
settings.validate()
|
||||
return settings
|
||||
|
||||
def validate(self) -> None:
|
||||
if self.project_id != EXPECTED_PROJECT:
|
||||
raise ValueError(f"GCP_PROJECT_ID must be {EXPECTED_PROJECT}")
|
||||
if self.instance_connection_name != EXPECTED_CONNECTION_NAME:
|
||||
raise ValueError(f"CLOUD_SQL_INSTANCE must be {EXPECTED_CONNECTION_NAME}")
|
||||
if self.database != EXPECTED_DATABASE:
|
||||
raise ValueError(f"DB_NAME must be {EXPECTED_DATABASE}")
|
||||
if self.authorization_role != EXPECTED_AUTHORIZATION_ROLE:
|
||||
raise ValueError(f"DB_AUTHORIZATION_ROLE must be {EXPECTED_AUTHORIZATION_ROLE}")
|
||||
if not self.iam_database_user.endswith(f"@{EXPECTED_PROJECT}.iam"):
|
||||
raise ValueError("DB_IAM_USER must be the scoped Cloud SQL service-account user")
|
||||
if any(character.isspace() for character in self.iam_database_user):
|
||||
raise ValueError("DB_IAM_USER must not contain whitespace")
|
||||
if (
|
||||
len(self.api_key) < 32
|
||||
or len(self.api_key) > 512
|
||||
or not self.api_key.isascii()
|
||||
or any(character.isspace() for character in self.api_key)
|
||||
):
|
||||
raise ValueError("OBSERVATORY_API_KEY must be 32-512 non-whitespace ASCII characters")
|
||||
if not 1 <= self.port <= 65535:
|
||||
raise ValueError("PORT is outside the valid TCP port range")
|
||||
315
observatory_read_adapter/db.py
Normal file
315
observatory_read_adapter/db.py
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import date, datetime, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from .config import Settings
|
||||
|
||||
ConnectionFactory = Callable[[], Any]
|
||||
|
||||
|
||||
class ReaderContractError(RuntimeError):
|
||||
"""The live database session does not match the adapter's read-only contract."""
|
||||
|
||||
|
||||
def _json_value(value: Any) -> Any:
|
||||
if isinstance(value, Decimal):
|
||||
return float(value)
|
||||
if isinstance(value, (date, datetime)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, tuple):
|
||||
return list(value)
|
||||
return value
|
||||
|
||||
|
||||
def _column_name(column: Any) -> str:
|
||||
return str(column.name) if hasattr(column, "name") else str(column[0])
|
||||
|
||||
|
||||
def _row_dict(cursor: Any, row: Any) -> dict[str, Any] | None:
|
||||
if row is None:
|
||||
return None
|
||||
names = [_column_name(column) for column in cursor.description]
|
||||
return {name: _json_value(value) for name, value in zip(names, row, strict=True)}
|
||||
|
||||
|
||||
def _rows(cursor: Any) -> list[dict[str, Any]]:
|
||||
names = [_column_name(column) for column in cursor.description]
|
||||
return [
|
||||
{name: _json_value(value) for name, value in zip(names, row, strict=True)}
|
||||
for row in cursor.fetchall()
|
||||
]
|
||||
|
||||
|
||||
class CloudSqlClaimReader:
|
||||
def __init__(self, settings: Settings, *, connection_factory: ConnectionFactory | None = None):
|
||||
self.settings = settings
|
||||
self._connector: Any | None = None
|
||||
if connection_factory is not None:
|
||||
self._connection_factory = connection_factory
|
||||
return
|
||||
|
||||
from google.cloud.sql.connector import Connector, IPTypes
|
||||
|
||||
self._connector = Connector(refresh_strategy="LAZY")
|
||||
|
||||
def connect() -> Any:
|
||||
return self._connector.connect(
|
||||
settings.instance_connection_name,
|
||||
"pg8000",
|
||||
user=settings.iam_database_user,
|
||||
db=settings.database,
|
||||
enable_iam_auth=True,
|
||||
ip_type=IPTypes.PRIVATE,
|
||||
)
|
||||
|
||||
self._connection_factory = connect
|
||||
|
||||
def close(self) -> None:
|
||||
if self._connector is not None:
|
||||
self._connector.close()
|
||||
|
||||
def check_ready(self) -> dict[str, Any]:
|
||||
connection = self._connection_factory()
|
||||
try:
|
||||
cursor = connection.cursor()
|
||||
self._begin_read_only(cursor)
|
||||
session = self._load_session_contract(cursor)
|
||||
connection.rollback()
|
||||
return {
|
||||
"ready": True,
|
||||
"database": session["database"],
|
||||
"authorization_role": self.settings.authorization_role,
|
||||
"transaction_read_only": True,
|
||||
}
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def read_claim(self, claim_id: str | None = None) -> dict[str, Any] | None:
|
||||
connection = self._connection_factory()
|
||||
try:
|
||||
cursor = connection.cursor()
|
||||
self._begin_read_only(cursor)
|
||||
session = self._load_session_contract(cursor)
|
||||
claim = self._load_claim(cursor, claim_id)
|
||||
if claim is None:
|
||||
connection.rollback()
|
||||
return None
|
||||
|
||||
evidence = self._load_evidence(cursor, claim["id"])
|
||||
edges = self._load_edges(cursor, claim["id"])
|
||||
proposal_counts = self._load_proposal_counts(cursor)
|
||||
related_proposals = self._load_related_proposals(cursor, claim["id"])
|
||||
recent_proposals = self._load_recent_proposals(cursor)
|
||||
connection.rollback()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
generated_at = datetime.now(timezone.utc).isoformat()
|
||||
return {
|
||||
"schema": "livingip.observatory-canonical-claim.v1",
|
||||
"generated_at": generated_at,
|
||||
"read_only": True,
|
||||
"provenance": {
|
||||
"gcp_project": self.settings.project_id,
|
||||
"cloud_sql_instance": self.settings.instance_connection_name,
|
||||
"database": session["database"],
|
||||
"database_principal": session["database_principal"],
|
||||
"authorization_role": self.settings.authorization_role,
|
||||
"transaction_read_only": True,
|
||||
"write_privileges_denied": bool(session["required_writes_denied"]),
|
||||
"service_revision": self.settings.service_revision,
|
||||
},
|
||||
"canonical": {
|
||||
"relation": "public.claims",
|
||||
"claim": claim,
|
||||
"evidence_relation": "public.claim_evidence -> public.sources",
|
||||
"evidence": evidence,
|
||||
"edge_relation": "public.claim_edges",
|
||||
"edges": edges,
|
||||
},
|
||||
"proposal_ledger": {
|
||||
"relation": "kb_stage.kb_proposals",
|
||||
"distinct_from_canonical": True,
|
||||
"status_counts": proposal_counts,
|
||||
"related": related_proposals,
|
||||
"recent": recent_proposals,
|
||||
"semantics": {
|
||||
"pending_review": "staged only; not canonical",
|
||||
"approved": "reviewed intent; not canonical until applied_at is set",
|
||||
"applied": "proposal receipt only; canonical rows remain authoritative",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _begin_read_only(cursor: Any) -> None:
|
||||
cursor.execute("set transaction read only")
|
||||
cursor.execute("set local statement_timeout = '5000ms'")
|
||||
cursor.execute("set local lock_timeout = '1000ms'")
|
||||
|
||||
def _load_session_contract(self, cursor: Any) -> dict[str, Any]:
|
||||
cursor.execute(
|
||||
"""
|
||||
select current_database() as database,
|
||||
current_user as database_principal,
|
||||
current_setting('transaction_read_only') as transaction_read_only,
|
||||
pg_has_role(current_user, %s, 'member') as has_authorization_role,
|
||||
has_table_privilege(current_user, 'public.claims', 'select')
|
||||
and has_table_privilege(current_user, 'public.sources', 'select')
|
||||
and has_table_privilege(current_user, 'public.claim_evidence', 'select')
|
||||
and has_table_privilege(current_user, 'public.claim_edges', 'select')
|
||||
and has_table_privilege(current_user, 'kb_stage.kb_proposals', 'select')
|
||||
as has_required_reads,
|
||||
not (
|
||||
has_table_privilege(current_user, 'public.claims', 'insert,update,delete,truncate')
|
||||
or has_table_privilege(current_user, 'public.sources', 'insert,update,delete,truncate')
|
||||
or has_table_privilege(current_user, 'public.claim_evidence', 'insert,update,delete,truncate')
|
||||
or has_table_privilege(current_user, 'public.claim_edges', 'insert,update,delete,truncate')
|
||||
or has_table_privilege(current_user, 'kb_stage.kb_proposals', 'insert,update,delete,truncate')
|
||||
) as required_writes_denied
|
||||
""",
|
||||
(self.settings.authorization_role,),
|
||||
)
|
||||
session = _row_dict(cursor, cursor.fetchone())
|
||||
if session is None:
|
||||
raise ReaderContractError("database session contract returned no row")
|
||||
if session["database"] != self.settings.database:
|
||||
raise ReaderContractError("database session selected an unexpected database")
|
||||
if session["database_principal"] != self.settings.iam_database_user:
|
||||
raise ReaderContractError("database session selected an unexpected principal")
|
||||
if session["transaction_read_only"] != "on":
|
||||
raise ReaderContractError("database session is not read-only")
|
||||
if not session["has_authorization_role"] or not session["has_required_reads"]:
|
||||
raise ReaderContractError("database principal lacks the exact read authorization")
|
||||
if not session["required_writes_denied"]:
|
||||
raise ReaderContractError("database principal has an unexpected write privilege")
|
||||
return session
|
||||
|
||||
@staticmethod
|
||||
def _load_claim(cursor: Any, claim_id: str | None) -> dict[str, Any] | None:
|
||||
columns = """
|
||||
select c.id::text as id,
|
||||
c.type::text as type,
|
||||
left(c.text, 12000) as text,
|
||||
c.status::text as status,
|
||||
c.confidence,
|
||||
coalesce(c.tags, '{}'::text[]) as tags,
|
||||
c.superseded_by::text as superseded_by,
|
||||
c.created_at,
|
||||
c.updated_at
|
||||
from public.claims c
|
||||
"""
|
||||
if claim_id is not None:
|
||||
cursor.execute(columns + " where c.id = %s::uuid", (claim_id,))
|
||||
return _row_dict(cursor, cursor.fetchone())
|
||||
|
||||
cursor.execute(
|
||||
columns
|
||||
+ """
|
||||
where exists (
|
||||
select 1 from public.claim_evidence ce where ce.claim_id = c.id
|
||||
)
|
||||
order by coalesce(c.updated_at, c.created_at) desc, c.id desc
|
||||
limit 1
|
||||
"""
|
||||
)
|
||||
claim = _row_dict(cursor, cursor.fetchone())
|
||||
if claim is not None:
|
||||
return claim
|
||||
cursor.execute(columns + " order by coalesce(c.updated_at, c.created_at) desc, c.id desc limit 1")
|
||||
return _row_dict(cursor, cursor.fetchone())
|
||||
|
||||
@staticmethod
|
||||
def _load_evidence(cursor: Any, claim_id: str) -> list[dict[str, Any]]:
|
||||
cursor.execute(
|
||||
"""
|
||||
select ce.id::text as evidence_id,
|
||||
ce.role::text as role,
|
||||
ce.weight,
|
||||
ce.created_at as linked_at,
|
||||
s.id::text as source_id,
|
||||
s.source_type::text as source_type,
|
||||
s.url,
|
||||
s.storage_path,
|
||||
left(coalesce(s.excerpt, ''), 2400) as excerpt,
|
||||
s.hash as content_hash,
|
||||
s.captured_at,
|
||||
s.created_at as source_created_at
|
||||
from public.claim_evidence ce
|
||||
join public.sources s on s.id = ce.source_id
|
||||
where ce.claim_id = %s::uuid
|
||||
order by ce.weight desc nulls last, ce.created_at, ce.id
|
||||
limit 30
|
||||
""",
|
||||
(claim_id,),
|
||||
)
|
||||
return _rows(cursor)
|
||||
|
||||
@staticmethod
|
||||
def _load_edges(cursor: Any, claim_id: str) -> list[dict[str, Any]]:
|
||||
cursor.execute(
|
||||
"""
|
||||
select e.id::text as edge_id,
|
||||
case when e.from_claim = %s::uuid then 'outgoing' else 'incoming' end as direction,
|
||||
e.edge_type::text as edge_type,
|
||||
e.weight,
|
||||
case when e.from_claim = %s::uuid then e.to_claim::text else e.from_claim::text end
|
||||
as connected_claim_id,
|
||||
e.created_at
|
||||
from public.claim_edges e
|
||||
where e.from_claim = %s::uuid or e.to_claim = %s::uuid
|
||||
order by e.created_at, e.id
|
||||
limit 40
|
||||
""",
|
||||
(claim_id, claim_id, claim_id, claim_id),
|
||||
)
|
||||
return _rows(cursor)
|
||||
|
||||
@staticmethod
|
||||
def _load_proposal_counts(cursor: Any) -> dict[str, int]:
|
||||
cursor.execute(
|
||||
"select status::text as status, count(*)::int as count "
|
||||
"from kb_stage.kb_proposals group by status order by status"
|
||||
)
|
||||
return {str(status): int(count) for status, count in cursor.fetchall()}
|
||||
|
||||
@staticmethod
|
||||
def _load_related_proposals(cursor: Any, claim_id: str) -> list[dict[str, Any]]:
|
||||
cursor.execute(
|
||||
"""
|
||||
select p.id::text as id,
|
||||
p.proposal_type::text as proposal_type,
|
||||
p.status::text as status,
|
||||
to_jsonb(p)->>'source_ref' as source_ref,
|
||||
p.reviewed_at,
|
||||
p.applied_at,
|
||||
p.updated_at
|
||||
from kb_stage.kb_proposals p
|
||||
where p.payload::text like %s
|
||||
order by p.updated_at desc, p.id desc
|
||||
limit 20
|
||||
""",
|
||||
(f"%{claim_id}%",),
|
||||
)
|
||||
return _rows(cursor)
|
||||
|
||||
@staticmethod
|
||||
def _load_recent_proposals(cursor: Any) -> list[dict[str, Any]]:
|
||||
cursor.execute(
|
||||
"""
|
||||
select p.id::text as id,
|
||||
p.proposal_type::text as proposal_type,
|
||||
p.status::text as status,
|
||||
to_jsonb(p)->>'source_ref' as source_ref,
|
||||
p.reviewed_at,
|
||||
p.applied_at,
|
||||
p.updated_at
|
||||
from kb_stage.kb_proposals p
|
||||
order by p.updated_at desc, p.id desc
|
||||
limit 10
|
||||
"""
|
||||
)
|
||||
return _rows(cursor)
|
||||
183
ops/observatory_read_role.sql
Normal file
183
ops/observatory_read_role.sql
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
\set ON_ERROR_STOP on
|
||||
\getenv observatory_iam_user OBSERVATORY_DB_IAM_USER
|
||||
|
||||
-- Provision only after Cloud SQL has created the dedicated IAM database user.
|
||||
-- The service-account username omits the .gserviceaccount.com suffix.
|
||||
select set_config('observatory.iam_user', :'observatory_iam_user', false);
|
||||
|
||||
do $preflight$
|
||||
declare
|
||||
principal text := current_setting('observatory.iam_user');
|
||||
begin
|
||||
if current_database() <> 'teleo_canonical' then
|
||||
raise exception 'observatory_read_role must run only in teleo_canonical';
|
||||
end if;
|
||||
if principal !~ '^[a-z0-9-]+@teleo-501523[.]iam$' then
|
||||
raise exception 'OBSERVATORY_DB_IAM_USER is not the scoped GCP service-account database user';
|
||||
end if;
|
||||
if not exists (select 1 from pg_catalog.pg_roles where rolname = principal) then
|
||||
raise exception 'Cloud SQL IAM database user does not exist: %', principal;
|
||||
end if;
|
||||
end
|
||||
$preflight$;
|
||||
|
||||
select 'create role kb_observatory_read nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1'
|
||||
where not exists (select 1 from pg_catalog.pg_roles where rolname = 'kb_observatory_read')
|
||||
\gexec
|
||||
|
||||
alter role kb_observatory_read
|
||||
with nologin nosuperuser nocreatedb nocreaterole noinherit noreplication nobypassrls connection limit -1
|
||||
password null;
|
||||
alter role kb_observatory_read reset all;
|
||||
|
||||
revoke all privileges on database teleo_canonical from kb_observatory_read;
|
||||
grant connect on database teleo_canonical to kb_observatory_read;
|
||||
|
||||
revoke all on schema public, kb_stage from kb_observatory_read;
|
||||
revoke all privileges on all tables in schema public, kb_stage from kb_observatory_read;
|
||||
revoke all privileges on all sequences in schema public, kb_stage from kb_observatory_read;
|
||||
revoke all privileges on all functions in schema public, kb_stage from kb_observatory_read;
|
||||
revoke all privileges on all procedures in schema public, kb_stage from kb_observatory_read;
|
||||
|
||||
select format(
|
||||
'revoke all privileges (%s) on table %I.%I from kb_observatory_read',
|
||||
pg_catalog.string_agg(pg_catalog.quote_ident(attribute.attname), ', ' order by attribute.attnum),
|
||||
namespace.nspname,
|
||||
relation.relname
|
||||
)
|
||||
from pg_catalog.pg_class relation
|
||||
join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace
|
||||
join pg_catalog.pg_attribute attribute on attribute.attrelid = relation.oid
|
||||
where namespace.nspname in ('public', 'kb_stage')
|
||||
and relation.relkind in ('r', 'p', 'v', 'm', 'f')
|
||||
and attribute.attnum > 0
|
||||
and not attribute.attisdropped
|
||||
group by namespace.nspname, relation.relname
|
||||
\gexec
|
||||
|
||||
grant usage on schema public, kb_stage to kb_observatory_read;
|
||||
grant select on
|
||||
public.claims,
|
||||
public.sources,
|
||||
public.claim_evidence,
|
||||
public.claim_edges,
|
||||
kb_stage.kb_proposals
|
||||
to kb_observatory_read;
|
||||
|
||||
do $membership$
|
||||
declare
|
||||
principal text := current_setting('observatory.iam_user');
|
||||
unexpected text;
|
||||
begin
|
||||
select pg_catalog.string_agg(granted.rolname, ', ' order by granted.rolname)
|
||||
into unexpected
|
||||
from pg_catalog.pg_auth_members membership
|
||||
join pg_catalog.pg_roles granted on granted.oid = membership.roleid
|
||||
join pg_catalog.pg_roles member on member.oid = membership.member
|
||||
where member.rolname = principal
|
||||
and granted.rolname <> 'kb_observatory_read';
|
||||
if unexpected is not null then
|
||||
raise exception 'dedicated Observatory principal has unexpected role memberships: %', unexpected;
|
||||
end if;
|
||||
|
||||
select pg_catalog.string_agg(member.rolname, ', ' order by member.rolname)
|
||||
into unexpected
|
||||
from pg_catalog.pg_auth_members membership
|
||||
join pg_catalog.pg_roles granted on granted.oid = membership.roleid
|
||||
join pg_catalog.pg_roles member on member.oid = membership.member
|
||||
where granted.rolname = 'kb_observatory_read'
|
||||
and member.rolname <> principal;
|
||||
if unexpected is not null then
|
||||
raise exception 'kb_observatory_read has unexpected members: %', unexpected;
|
||||
end if;
|
||||
|
||||
execute pg_catalog.format('grant kb_observatory_read to %I', principal);
|
||||
execute pg_catalog.format(
|
||||
'alter role %I in database teleo_canonical set default_transaction_read_only = on',
|
||||
principal
|
||||
);
|
||||
execute pg_catalog.format(
|
||||
'alter role %I in database teleo_canonical set statement_timeout = %L',
|
||||
principal,
|
||||
'5s'
|
||||
);
|
||||
execute pg_catalog.format(
|
||||
'alter role %I in database teleo_canonical set lock_timeout = %L',
|
||||
principal,
|
||||
'1s'
|
||||
);
|
||||
end
|
||||
$membership$;
|
||||
|
||||
do $verify$
|
||||
declare
|
||||
principal text := current_setting('observatory.iam_user');
|
||||
unexpected text;
|
||||
begin
|
||||
if not pg_catalog.pg_has_role(principal, 'kb_observatory_read', 'member') then
|
||||
raise exception 'Observatory principal is not a member of kb_observatory_read';
|
||||
end if;
|
||||
if not exists (
|
||||
select 1 from pg_catalog.pg_roles
|
||||
where rolname = principal and rolinherit and not rolsuper and not rolcreatedb
|
||||
and not rolcreaterole and not rolreplication and not rolbypassrls
|
||||
) then
|
||||
raise exception 'Observatory principal has unsafe role attributes or cannot inherit read grants';
|
||||
end if;
|
||||
|
||||
select pg_catalog.string_agg(
|
||||
pg_catalog.format('%I.%I:%s', namespace.nspname, relation.relname, privilege.name),
|
||||
', ' order by namespace.nspname, relation.relname, privilege.name
|
||||
)
|
||||
into unexpected
|
||||
from pg_catalog.pg_class relation
|
||||
join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace
|
||||
cross join (values ('INSERT'), ('UPDATE'), ('DELETE'), ('TRUNCATE'), ('REFERENCES'), ('TRIGGER')) privilege(name)
|
||||
where namespace.nspname in ('public', 'kb_stage')
|
||||
and relation.relkind in ('r', 'p', 'v', 'm', 'f')
|
||||
and pg_catalog.has_table_privilege(principal, relation.oid, privilege.name);
|
||||
if unexpected is not null then
|
||||
raise exception 'Observatory principal has effective table writes: %', unexpected;
|
||||
end if;
|
||||
|
||||
select pg_catalog.string_agg(
|
||||
pg_catalog.format('%I.%I', namespace.nspname, relation.relname),
|
||||
', ' order by namespace.nspname, relation.relname
|
||||
)
|
||||
into unexpected
|
||||
from pg_catalog.pg_class relation
|
||||
join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace
|
||||
where namespace.nspname in ('public', 'kb_stage')
|
||||
and relation.relkind in ('r', 'p', 'v', 'm', 'f')
|
||||
and pg_catalog.has_table_privilege(principal, relation.oid, 'SELECT')
|
||||
and (namespace.nspname, relation.relname) not in (
|
||||
('public', 'claims'),
|
||||
('public', 'sources'),
|
||||
('public', 'claim_evidence'),
|
||||
('public', 'claim_edges'),
|
||||
('kb_stage', 'kb_proposals')
|
||||
);
|
||||
if unexpected is not null then
|
||||
raise exception 'Observatory principal can SELECT outside its allowlist: %', unexpected;
|
||||
end if;
|
||||
|
||||
if not (
|
||||
pg_catalog.has_table_privilege(principal, 'public.claims', 'SELECT')
|
||||
and pg_catalog.has_table_privilege(principal, 'public.sources', 'SELECT')
|
||||
and pg_catalog.has_table_privilege(principal, 'public.claim_evidence', 'SELECT')
|
||||
and pg_catalog.has_table_privilege(principal, 'public.claim_edges', 'SELECT')
|
||||
and pg_catalog.has_table_privilege(principal, 'kb_stage.kb_proposals', 'SELECT')
|
||||
) then
|
||||
raise exception 'Observatory principal is missing a required read grant';
|
||||
end if;
|
||||
end
|
||||
$verify$;
|
||||
|
||||
select jsonb_build_object(
|
||||
'database', current_database(),
|
||||
'authorization_role', 'kb_observatory_read',
|
||||
'database_principal', current_setting('observatory.iam_user'),
|
||||
'required_reads', true,
|
||||
'effective_table_writes_denied', true,
|
||||
'default_transaction_read_only', true
|
||||
)::text;
|
||||
|
|
@ -18,9 +18,12 @@ dev = [
|
|||
"pytest-asyncio>=0.23",
|
||||
"ruff>=0.3",
|
||||
]
|
||||
observatory = [
|
||||
"cloud-sql-python-connector[pg8000]>=1.20.4,<1.21",
|
||||
]
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["lib"]
|
||||
packages = ["lib", "observatory_read_adapter"]
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py311"
|
||||
|
|
|
|||
346
tests/test_observatory_read_adapter.py
Normal file
346
tests/test_observatory_read_adapter.py
Normal file
|
|
@ -0,0 +1,346 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp.test_utils import TestClient, TestServer
|
||||
|
||||
from observatory_read_adapter.app import create_app
|
||||
from observatory_read_adapter.config import EXPECTED_CONNECTION_NAME, Settings
|
||||
from observatory_read_adapter.db import CloudSqlClaimReader
|
||||
|
||||
CLAIM_ID = "d0000000-0000-0000-0000-000000000005"
|
||||
API_KEY = "observatory-test-api-key-000000000000000000000000"
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def settings(**overrides: object) -> Settings:
|
||||
values = {
|
||||
"project_id": "teleo-501523",
|
||||
"instance_connection_name": EXPECTED_CONNECTION_NAME,
|
||||
"database": "teleo_canonical",
|
||||
"iam_database_user": "sa-observatory-read-adapter@teleo-501523.iam",
|
||||
"authorization_role": "kb_observatory_read",
|
||||
"api_key": API_KEY,
|
||||
"service_revision": "test-revision",
|
||||
}
|
||||
values.update(overrides)
|
||||
return Settings(**values)
|
||||
|
||||
|
||||
class FakeReader:
|
||||
def __init__(self) -> None:
|
||||
self.closed = False
|
||||
|
||||
def check_ready(self) -> dict[str, object]:
|
||||
return {
|
||||
"ready": True,
|
||||
"database": "teleo_canonical",
|
||||
"authorization_role": "kb_observatory_read",
|
||||
"transaction_read_only": True,
|
||||
}
|
||||
|
||||
def read_claim(self, claim_id: str | None = None) -> dict[str, object] | None:
|
||||
if claim_id not in (None, CLAIM_ID):
|
||||
return None
|
||||
return {
|
||||
"schema": "livingip.observatory-canonical-claim.v1",
|
||||
"read_only": True,
|
||||
"provenance": {
|
||||
"database": "teleo_canonical",
|
||||
"database_principal": "sa-observatory-read-adapter@teleo-501523.iam",
|
||||
"authorization_role": "kb_observatory_read",
|
||||
"transaction_read_only": True,
|
||||
"write_privileges_denied": True,
|
||||
},
|
||||
"canonical": {
|
||||
"claim": {"id": CLAIM_ID, "status": "open", "text": "Canonical claim"},
|
||||
"evidence": [{"source_id": "e0000000-0000-0000-0000-000000000006"}],
|
||||
},
|
||||
"proposal_ledger": {
|
||||
"distinct_from_canonical": True,
|
||||
"status_counts": {"pending_review": 2, "approved": 1, "applied": 1},
|
||||
"related": [{"status": "approved", "applied_at": None}],
|
||||
},
|
||||
}
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self) -> None:
|
||||
self.description: list[tuple[str]] = []
|
||||
self.result: list[tuple[object, ...]] = []
|
||||
self.executed: list[tuple[str, tuple[object, ...]]] = []
|
||||
|
||||
def execute(self, sql: str, parameters: tuple[object, ...] = ()) -> None:
|
||||
normalized = " ".join(sql.split()).lower()
|
||||
self.executed.append((normalized, parameters))
|
||||
self.description = []
|
||||
self.result = []
|
||||
if "current_database() as database" in normalized:
|
||||
self.description = [
|
||||
("database",),
|
||||
("database_principal",),
|
||||
("transaction_read_only",),
|
||||
("has_authorization_role",),
|
||||
("has_required_reads",),
|
||||
("required_writes_denied",),
|
||||
]
|
||||
self.result = [
|
||||
(
|
||||
"teleo_canonical",
|
||||
"sa-observatory-read-adapter@teleo-501523.iam",
|
||||
"on",
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
)
|
||||
]
|
||||
elif "from public.claims c" in normalized:
|
||||
self.description = [
|
||||
("id",),
|
||||
("type",),
|
||||
("text",),
|
||||
("status",),
|
||||
("confidence",),
|
||||
("tags",),
|
||||
("superseded_by",),
|
||||
("created_at",),
|
||||
("updated_at",),
|
||||
]
|
||||
self.result = [
|
||||
(
|
||||
CLAIM_ID,
|
||||
"strategy",
|
||||
"Canonical claim",
|
||||
"open",
|
||||
0.8,
|
||||
["test"],
|
||||
None,
|
||||
"2026-07-15T00:00:00+00:00",
|
||||
"2026-07-15T00:00:00+00:00",
|
||||
)
|
||||
]
|
||||
elif "from public.claim_evidence ce" in normalized:
|
||||
self.description = [
|
||||
("evidence_id",),
|
||||
("role",),
|
||||
("weight",),
|
||||
("linked_at",),
|
||||
("source_id",),
|
||||
("source_type",),
|
||||
("url",),
|
||||
("storage_path",),
|
||||
("excerpt",),
|
||||
("content_hash",),
|
||||
("captured_at",),
|
||||
("source_created_at",),
|
||||
]
|
||||
self.result = [
|
||||
(
|
||||
"e0000000-0000-0000-0000-000000000006",
|
||||
"grounds",
|
||||
1.0,
|
||||
"2026-07-15T00:00:00+00:00",
|
||||
"f0000000-0000-0000-0000-000000000007",
|
||||
"document",
|
||||
"https://example.test/source",
|
||||
None,
|
||||
"Source excerpt",
|
||||
"sha256:test",
|
||||
"2026-07-15T00:00:00+00:00",
|
||||
"2026-07-15T00:00:00+00:00",
|
||||
)
|
||||
]
|
||||
elif "from public.claim_edges e" in normalized:
|
||||
self.description = [
|
||||
("edge_id",),
|
||||
("direction",),
|
||||
("edge_type",),
|
||||
("weight",),
|
||||
("connected_claim_id",),
|
||||
("created_at",),
|
||||
]
|
||||
self.result = []
|
||||
elif "group by status" in normalized:
|
||||
self.description = [("status",), ("count",)]
|
||||
self.result = [("approved", 1), ("applied", 1), ("pending_review", 2)]
|
||||
elif "where p.payload::text like" in normalized:
|
||||
self.description = [
|
||||
("id",),
|
||||
("proposal_type",),
|
||||
("status",),
|
||||
("source_ref",),
|
||||
("reviewed_at",),
|
||||
("applied_at",),
|
||||
("updated_at",),
|
||||
]
|
||||
self.result = [
|
||||
(
|
||||
"a0000000-0000-0000-0000-000000000001",
|
||||
"revise_claim",
|
||||
"approved",
|
||||
"source:test",
|
||||
"2026-07-15T00:00:00+00:00",
|
||||
None,
|
||||
"2026-07-15T00:00:00+00:00",
|
||||
)
|
||||
]
|
||||
elif "from kb_stage.kb_proposals p" in normalized:
|
||||
self.description = [
|
||||
("id",),
|
||||
("proposal_type",),
|
||||
("status",),
|
||||
("source_ref",),
|
||||
("reviewed_at",),
|
||||
("applied_at",),
|
||||
("updated_at",),
|
||||
]
|
||||
self.result = []
|
||||
|
||||
def fetchone(self) -> tuple[object, ...] | None:
|
||||
return self.result.pop(0) if self.result else None
|
||||
|
||||
def fetchall(self) -> list[tuple[object, ...]]:
|
||||
result, self.result = self.result, []
|
||||
return result
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(self) -> None:
|
||||
self.cursor_value = FakeCursor()
|
||||
self.rolled_back = False
|
||||
self.closed = False
|
||||
|
||||
def cursor(self) -> FakeCursor:
|
||||
return self.cursor_value
|
||||
|
||||
def rollback(self) -> None:
|
||||
self.rolled_back = True
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
def test_settings_fail_closed_on_wrong_database_role_or_short_key() -> None:
|
||||
for overrides in (
|
||||
{"database": "teleo"},
|
||||
{"authorization_role": "leoclean_kb_runtime"},
|
||||
{"api_key": "too-short"},
|
||||
{"instance_connection_name": "other:region:instance"},
|
||||
):
|
||||
candidate = settings(**overrides)
|
||||
try:
|
||||
candidate.validate()
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"unsafe settings accepted: {overrides}")
|
||||
|
||||
|
||||
def test_authenticated_claim_response_and_negative_http_paths() -> None:
|
||||
async def exercise() -> None:
|
||||
reader = FakeReader()
|
||||
client = TestClient(TestServer(create_app(settings(), reader=reader)))
|
||||
await client.start_server()
|
||||
try:
|
||||
anonymous = await client.get("/v1/claims/sample")
|
||||
assert anonymous.status == 401
|
||||
assert await anonymous.json() == {"error": "unauthorized"}
|
||||
|
||||
response = await client.get(
|
||||
f"/v1/claims/{CLAIM_ID}",
|
||||
headers={"X-Api-Key": API_KEY},
|
||||
)
|
||||
assert response.status == 200
|
||||
assert response.headers["Cache-Control"] == "private, no-store, max-age=0"
|
||||
payload = await response.json()
|
||||
assert payload["schema"] == "livingip.observatory-canonical-claim.v1"
|
||||
assert payload["provenance"]["database"] == "teleo_canonical"
|
||||
assert payload["provenance"]["write_privileges_denied"] is True
|
||||
assert payload["canonical"]["evidence"]
|
||||
assert payload["proposal_ledger"]["distinct_from_canonical"] is True
|
||||
assert payload["proposal_ledger"]["related"][0]["applied_at"] is None
|
||||
assert API_KEY not in json.dumps(payload)
|
||||
|
||||
write_attempt = await client.post(
|
||||
f"/v1/claims/{CLAIM_ID}",
|
||||
headers={"X-Api-Key": API_KEY},
|
||||
json={"status": "applied"},
|
||||
)
|
||||
assert write_attempt.status == 405
|
||||
|
||||
invalid = await client.get(
|
||||
"/v1/claims/not-a-uuid",
|
||||
headers={"X-Api-Key": API_KEY},
|
||||
)
|
||||
assert invalid.status == 400
|
||||
finally:
|
||||
await client.close()
|
||||
assert reader.closed is True
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
def test_readiness_checks_database_contract_without_exposing_claims() -> None:
|
||||
async def exercise() -> None:
|
||||
client = TestClient(TestServer(create_app(settings(), reader=FakeReader())))
|
||||
await client.start_server()
|
||||
try:
|
||||
response = await client.get("/readyz")
|
||||
payload = await response.json()
|
||||
assert response.status == 200
|
||||
assert payload == {"status": "ready"}
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
def test_database_reader_returns_provenance_and_keeps_proposals_distinct() -> None:
|
||||
connection = FakeConnection()
|
||||
reader = CloudSqlClaimReader(settings(), connection_factory=lambda: connection)
|
||||
|
||||
payload = reader.read_claim(CLAIM_ID)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["provenance"]["database"] == "teleo_canonical"
|
||||
assert payload["provenance"]["database_principal"] == settings().iam_database_user
|
||||
assert payload["provenance"]["write_privileges_denied"] is True
|
||||
assert payload["canonical"]["claim"]["id"] == CLAIM_ID
|
||||
assert payload["canonical"]["evidence"][0]["content_hash"] == "sha256:test"
|
||||
assert payload["proposal_ledger"]["distinct_from_canonical"] is True
|
||||
assert payload["proposal_ledger"]["related"][0]["status"] == "approved"
|
||||
assert payload["proposal_ledger"]["related"][0]["applied_at"] is None
|
||||
assert connection.rolled_back is True
|
||||
assert connection.closed is True
|
||||
assert any("set transaction read only" in sql for sql, _parameters in connection.cursor_value.executed)
|
||||
|
||||
|
||||
def test_database_role_contract_is_select_only_and_iam_scoped() -> None:
|
||||
sql = (ROOT / "ops" / "observatory_read_role.sql").read_text(encoding="utf-8")
|
||||
lowered = sql.lower()
|
||||
|
||||
assert "kb_observatory_read nologin" in lowered
|
||||
assert "default_transaction_read_only = on" in lowered
|
||||
assert "public.claims" in lowered
|
||||
assert "public.sources" in lowered
|
||||
assert "public.claim_evidence" in lowered
|
||||
assert "public.claim_edges" in lowered
|
||||
assert "kb_stage.kb_proposals" in lowered
|
||||
assert "grant select on" in lowered
|
||||
assert "grant insert" not in lowered
|
||||
assert "grant update" not in lowered
|
||||
assert "grant delete" not in lowered
|
||||
assert "effective_table_writes_denied" in lowered
|
||||
|
||||
|
||||
def test_container_runs_as_non_root_and_contains_no_password_contract() -> None:
|
||||
dockerfile = (ROOT / "Dockerfile.observatory-read-adapter").read_text(encoding="utf-8")
|
||||
|
||||
assert "USER 10001:10001" in dockerfile
|
||||
assert "PGPASSWORD" not in dockerfile
|
||||
assert "DB_PASS" not in dockerfile
|
||||
182
tests/test_observatory_read_role_postgres.py
Normal file
182
tests/test_observatory_read_role_postgres.py
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
ROLE_SQL = ROOT / "ops" / "observatory_read_role.sql"
|
||||
POSTGRES_IMAGE = "postgres:16-alpine"
|
||||
DATABASE = "teleo_canonical"
|
||||
IAM_USER = "sa-observatory-read-adapter@teleo-501523.iam"
|
||||
TEST_PASSWORD = "generated-test-only-password"
|
||||
|
||||
BOOTSTRAP_SQL = f"""
|
||||
begin;
|
||||
create schema kb_stage;
|
||||
create role "{IAM_USER}" login inherit password '{TEST_PASSWORD}';
|
||||
create table public.claims (id uuid primary key);
|
||||
create table public.sources (id uuid primary key);
|
||||
create table public.claim_evidence (id uuid primary key);
|
||||
create table public.claim_edges (id uuid primary key);
|
||||
create table public.private_notes (id uuid primary key);
|
||||
create table kb_stage.kb_proposals (id uuid primary key);
|
||||
insert into public.claims values ('11111111-1111-1111-1111-111111111111');
|
||||
commit;
|
||||
"""
|
||||
|
||||
|
||||
def run(command: list[str], *, input_text: str | None = None, check: bool = True) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
command,
|
||||
input=input_text,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=check,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker is required")
|
||||
def test_observatory_role_is_read_only_and_exactly_allowlisted() -> None:
|
||||
container = f"teleo-observatory-role-{uuid.uuid4().hex[:10]}"
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"--detach",
|
||||
"--rm",
|
||||
"--name",
|
||||
container,
|
||||
"--env",
|
||||
f"POSTGRES_PASSWORD={TEST_PASSWORD}",
|
||||
"--env",
|
||||
f"POSTGRES_DB={DATABASE}",
|
||||
POSTGRES_IMAGE,
|
||||
]
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + 25
|
||||
while time.monotonic() < deadline:
|
||||
ready = run(
|
||||
["docker", "exec", container, "pg_isready", "-U", "postgres", "-d", DATABASE],
|
||||
check=False,
|
||||
)
|
||||
if ready.returncode == 0:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
else:
|
||||
raise AssertionError("ephemeral Postgres did not become ready")
|
||||
|
||||
bootstrap: subprocess.CompletedProcess[str] | None = None
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
bootstrap = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"-i",
|
||||
container,
|
||||
"psql",
|
||||
"--set",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-U",
|
||||
"postgres",
|
||||
"-d",
|
||||
DATABASE,
|
||||
],
|
||||
input_text=BOOTSTRAP_SQL,
|
||||
check=False,
|
||||
)
|
||||
if bootstrap.returncode == 0:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
else:
|
||||
assert bootstrap is not None
|
||||
raise AssertionError(f"ephemeral Postgres bootstrap failed: {bootstrap.stderr[-1000:]}")
|
||||
migration = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"-e",
|
||||
f"OBSERVATORY_DB_IAM_USER={IAM_USER}",
|
||||
"-i",
|
||||
container,
|
||||
"psql",
|
||||
"-U",
|
||||
"postgres",
|
||||
"-d",
|
||||
DATABASE,
|
||||
],
|
||||
input_text=ROLE_SQL.read_text(encoding="utf-8"),
|
||||
)
|
||||
assert '"effective_table_writes_denied": true' in migration.stdout
|
||||
|
||||
readback = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"-e",
|
||||
f"PGPASSWORD={TEST_PASSWORD}",
|
||||
container,
|
||||
"psql",
|
||||
"-h",
|
||||
"127.0.0.1",
|
||||
"-U",
|
||||
IAM_USER,
|
||||
"-d",
|
||||
DATABASE,
|
||||
"-At",
|
||||
"-c",
|
||||
"show default_transaction_read_only; select count(*) from public.claims;",
|
||||
]
|
||||
)
|
||||
assert readback.stdout.splitlines() == ["on", "1"]
|
||||
|
||||
write_attempt = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"-e",
|
||||
f"PGPASSWORD={TEST_PASSWORD}",
|
||||
container,
|
||||
"psql",
|
||||
"-h",
|
||||
"127.0.0.1",
|
||||
"-U",
|
||||
IAM_USER,
|
||||
"-d",
|
||||
DATABASE,
|
||||
"-c",
|
||||
"insert into public.claims values ('22222222-2222-2222-2222-222222222222');",
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
assert write_attempt.returncode != 0
|
||||
|
||||
outside_allowlist = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"-e",
|
||||
f"PGPASSWORD={TEST_PASSWORD}",
|
||||
container,
|
||||
"psql",
|
||||
"-h",
|
||||
"127.0.0.1",
|
||||
"-U",
|
||||
IAM_USER,
|
||||
"-d",
|
||||
DATABASE,
|
||||
"-c",
|
||||
"select * from public.private_notes;",
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
assert outside_allowlist.returncode != 0
|
||||
finally:
|
||||
run(["docker", "rm", "--force", container], check=False)
|
||||
Loading…
Reference in a new issue