994 lines
42 KiB
Python
994 lines
42 KiB
Python
#!/usr/bin/env python3
|
|
"""Fail-closed, read-only VPS/Cloud SQL source fingerprint comparison."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import ipaddress
|
|
import json
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
from collections.abc import Callable
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
try:
|
|
from .private_receipt_io import print_private_receipt_summary, write_private_json
|
|
from .verify_postgres_parity_manifest import (
|
|
REVIEWED_MANIFEST_SQL_SHA256,
|
|
SINGLETON_KINDS,
|
|
canonical_hash,
|
|
compare_manifests,
|
|
)
|
|
except ImportError: # pragma: no cover - direct script execution
|
|
from private_receipt_io import print_private_receipt_summary, write_private_json
|
|
from verify_postgres_parity_manifest import (
|
|
REVIEWED_MANIFEST_SQL_SHA256,
|
|
SINGLETON_KINDS,
|
|
canonical_hash,
|
|
compare_manifests,
|
|
)
|
|
|
|
|
|
RECEIPT_SCHEMA = "livingip.vpsGcpSourceDrift.v1"
|
|
OBSERVATION_SCHEMA = "livingip.sourceFingerprintObservation.v1"
|
|
REMOTE_ROUTE_SCHEMA = "livingip.remoteSourceRoute.v1"
|
|
SAFE_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:@/+\-]{0,255}\Z")
|
|
COMMIT_RE = re.compile(r"[0-9a-f]{40}\Z")
|
|
ROWSET_MD5_RE = re.compile(r"[0-9a-f]{32}\Z")
|
|
ROUTE_PREFIX = "LEO_SOURCE_ROUTE\t"
|
|
EXIT_CODE_BY_STATUS = {"identical": 0, "drift": 1, "incomplete": 2, "invalid": 3}
|
|
|
|
|
|
class DriftDetectorError(RuntimeError):
|
|
"""Raised for an invalid detector configuration or observation."""
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def sha256_bytes(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def safe_value(value: str, flag: str) -> str:
|
|
if not SAFE_NAME_RE.fullmatch(value):
|
|
raise DriftDetectorError(f"{flag} contains an unsafe value")
|
|
return value
|
|
|
|
|
|
def same_network_address(left: Any, right: Any) -> bool:
|
|
if left is None or right is None:
|
|
return left is right
|
|
try:
|
|
return ipaddress.ip_address(str(left)) == ipaddress.ip_address(str(right))
|
|
except ValueError:
|
|
return str(left) == str(right)
|
|
|
|
|
|
def parse_manifest_lines(lines: list[str], label: str) -> dict[str, Any]:
|
|
singleton: dict[str, dict[str, Any]] = {}
|
|
tables: dict[str, dict[str, Any]] = {}
|
|
performance: dict[str, dict[str, Any]] = {}
|
|
for line_number, raw_line in enumerate(lines, 1):
|
|
line = raw_line.strip()
|
|
if not line or line in {"BEGIN", "SET", "ROLLBACK"}:
|
|
continue
|
|
try:
|
|
row = json.loads(line)
|
|
except json.JSONDecodeError as exc:
|
|
raise DriftDetectorError(f"{label} manifest line {line_number} is not JSON") from exc
|
|
if not isinstance(row, dict):
|
|
raise DriftDetectorError(f"{label} manifest line {line_number} is not an object")
|
|
kind = row.get("kind")
|
|
if kind in SINGLETON_KINDS:
|
|
if kind in singleton:
|
|
raise DriftDetectorError(f"{label} manifest repeats {kind}")
|
|
singleton[str(kind)] = row
|
|
elif kind == "table":
|
|
key = f"{row.get('schema')}.{row.get('table')}"
|
|
if key in tables:
|
|
raise DriftDetectorError(f"{label} manifest repeats table {key}")
|
|
tables[key] = row
|
|
elif kind == "performance":
|
|
key = str(row.get("query"))
|
|
if key in performance:
|
|
raise DriftDetectorError(f"{label} manifest repeats performance row {key}")
|
|
performance[key] = row
|
|
else:
|
|
raise DriftDetectorError(f"{label} manifest has unknown kind {kind!r}")
|
|
missing = SINGLETON_KINDS - set(singleton)
|
|
if missing:
|
|
raise DriftDetectorError(f"{label} manifest is missing {sorted(missing)}")
|
|
if not tables:
|
|
raise DriftDetectorError(f"{label} manifest has no table rows")
|
|
manifest = {"singleton": singleton, "tables": tables, "performance": performance}
|
|
validate_manifest_contract(manifest, label)
|
|
return manifest
|
|
|
|
|
|
def validate_manifest_contract(manifest: dict[str, Any], label: str) -> None:
|
|
singleton = manifest.get("singleton")
|
|
tables = manifest.get("tables")
|
|
performance = manifest.get("performance")
|
|
if not all(isinstance(item, dict) for item in (singleton, tables, performance)):
|
|
raise DriftDetectorError(f"{label} manifest sections are invalid")
|
|
if set(singleton) != SINGLETON_KINDS:
|
|
raise DriftDetectorError(f"{label} manifest singleton set is invalid")
|
|
identity = singleton["identity"]
|
|
if not isinstance(identity, dict) or identity.get("kind") != "identity":
|
|
raise DriftDetectorError(f"{label} manifest identity row is invalid")
|
|
for field in ("database", "current_user"):
|
|
if not isinstance(identity.get(field), str) or not identity[field]:
|
|
raise DriftDetectorError(f"{label} manifest identity {field} is invalid")
|
|
server_version = identity.get("server_version_num")
|
|
if not isinstance(server_version, int) or isinstance(server_version, bool) or server_version <= 0:
|
|
raise DriftDetectorError(f"{label} manifest server version is invalid")
|
|
if identity.get("transaction_read_only") not in {"on", "off"}:
|
|
raise DriftDetectorError(f"{label} manifest transaction mode is invalid")
|
|
address = identity.get("server_address")
|
|
port = identity.get("server_port")
|
|
if address is None:
|
|
if port is not None:
|
|
raise DriftDetectorError(f"{label} manifest database socket identity is invalid")
|
|
elif (
|
|
not isinstance(address, str)
|
|
or not address
|
|
or not isinstance(port, int)
|
|
or isinstance(port, bool)
|
|
or not 1 <= port <= 65535
|
|
):
|
|
raise DriftDetectorError(f"{label} manifest database socket identity is invalid")
|
|
if not isinstance(identity.get("ssl"), bool):
|
|
raise DriftDetectorError(f"{label} manifest TLS identity is invalid")
|
|
for kind in SINGLETON_KINDS - {"identity"}:
|
|
row = singleton[kind]
|
|
if not isinstance(row, dict) or row.get("kind") != kind or not isinstance(row.get("items"), list):
|
|
raise DriftDetectorError(f"{label} manifest {kind} row is invalid")
|
|
if not tables:
|
|
raise DriftDetectorError(f"{label} manifest has no table rows")
|
|
for key, row in tables.items():
|
|
if not isinstance(row, dict) or row.get("kind") != "table":
|
|
raise DriftDetectorError(f"{label} manifest table row is invalid")
|
|
schema = row.get("schema")
|
|
table = row.get("table")
|
|
row_count = row.get("row_count")
|
|
if not isinstance(schema, str) or not schema or not isinstance(table, str) or not table:
|
|
raise DriftDetectorError(f"{label} manifest table identity is invalid")
|
|
if key != f"{schema}.{table}":
|
|
raise DriftDetectorError(f"{label} manifest table key is detached from its row")
|
|
if not isinstance(row_count, int) or isinstance(row_count, bool) or row_count < 0:
|
|
raise DriftDetectorError(f"{label} manifest table row count is invalid")
|
|
if not isinstance(row.get("rowset_md5"), str) or not ROWSET_MD5_RE.fullmatch(row["rowset_md5"]):
|
|
raise DriftDetectorError(f"{label} manifest table row hash is invalid")
|
|
for key, row in performance.items():
|
|
if not isinstance(row, dict) or row.get("kind") != "performance" or row.get("query") != key:
|
|
raise DriftDetectorError(f"{label} manifest performance row is invalid")
|
|
elapsed_ms = row.get("elapsed_ms")
|
|
observed_rows = row.get("observed_rows")
|
|
if (
|
|
not isinstance(elapsed_ms, int | float)
|
|
or isinstance(elapsed_ms, bool)
|
|
or elapsed_ms < 0
|
|
or not isinstance(observed_rows, int)
|
|
or isinstance(observed_rows, bool)
|
|
or observed_rows < 0
|
|
):
|
|
raise DriftDetectorError(f"{label} manifest performance metrics are invalid")
|
|
|
|
|
|
def manifest_fingerprint(manifest: dict[str, Any]) -> dict[str, Any]:
|
|
tables = manifest["tables"]
|
|
table_rows = [tables[key] for key in sorted(tables)]
|
|
structure = {kind: manifest["singleton"][kind].get("items", []) for kind in sorted(SINGLETON_KINDS - {"identity"})}
|
|
return {
|
|
"table_count": len(tables),
|
|
"total_rows": sum(int(row["row_count"]) for row in table_rows),
|
|
"table_fingerprint_sha256": canonical_hash(table_rows),
|
|
"structure_fingerprint_sha256": canonical_hash(structure),
|
|
}
|
|
|
|
|
|
def validate_route_binding(
|
|
label: str,
|
|
route: dict[str, Any],
|
|
expected_target: dict[str, Any],
|
|
) -> None:
|
|
if route.get("schema") != REMOTE_ROUTE_SCHEMA:
|
|
raise DriftDetectorError(f"{label} route schema is invalid")
|
|
bindings = {
|
|
"route_kind": "route_kind",
|
|
"transport": "transport",
|
|
"service": "runtime_service",
|
|
"expected_database": "database",
|
|
"database_endpoint": "database_endpoint",
|
|
"database_user": "database_user",
|
|
}
|
|
for route_key, target_key in bindings.items():
|
|
expected = expected_target.get(target_key)
|
|
if expected is None or route.get(route_key) != expected:
|
|
raise DriftDetectorError(f"{label} observed route {route_key} does not match the explicit target")
|
|
endpoint_identity = route.get("database_endpoint_identity")
|
|
if expected_target.get("route_kind") == "vps_docker":
|
|
if not isinstance(endpoint_identity, dict) or endpoint_identity.get("kind") != "docker_container":
|
|
raise DriftDetectorError(f"{label} observed Docker endpoint identity is missing")
|
|
container_id = endpoint_identity.get("container_id")
|
|
container_name = endpoint_identity.get("container_name")
|
|
selector = str(expected_target["database_endpoint"]).removeprefix("docker:")
|
|
if (
|
|
not isinstance(container_id, str)
|
|
or not re.fullmatch(r"[0-9a-f]{64}", container_id)
|
|
or not isinstance(container_name, str)
|
|
or selector not in {container_id, container_id[:12], container_name}
|
|
or endpoint_identity.get("running") is not True
|
|
):
|
|
raise DriftDetectorError(f"{label} observed Docker endpoint identity is invalid")
|
|
elif endpoint_identity is not None:
|
|
raise DriftDetectorError(f"{label} route contains an unexpected endpoint identity")
|
|
hostname = route.get("hostname")
|
|
if not isinstance(hostname, str) or not SAFE_NAME_RE.fullmatch(hostname):
|
|
raise DriftDetectorError(f"{label} observed route hostname is invalid")
|
|
process_cwd = route.get("process_cwd")
|
|
if not isinstance(process_cwd, str) or not Path(process_cwd).is_absolute():
|
|
raise DriftDetectorError(f"{label} observed route process cwd is invalid")
|
|
service_state = route.get("service_state")
|
|
if not isinstance(service_state, dict):
|
|
raise DriftDetectorError(f"{label} observed route service state is missing")
|
|
try:
|
|
main_pid = int(service_state.get("MainPID") or 0)
|
|
except (TypeError, ValueError) as exc:
|
|
raise DriftDetectorError(f"{label} observed route MainPID is invalid") from exc
|
|
if service_state.get("ActiveState") != "active" or service_state.get("SubState") != "running" or main_pid <= 0:
|
|
raise DriftDetectorError(f"{label} observed route service is not active/running")
|
|
runtime_commit = route.get("runtime_commit")
|
|
if not isinstance(runtime_commit, str) or not COMMIT_RE.fullmatch(runtime_commit):
|
|
raise DriftDetectorError(f"{label} runtime commit identity is missing or ambiguous")
|
|
candidates = route.get("commit_candidates")
|
|
if route.get("commit_identity_ambiguous") is not False or not isinstance(candidates, dict):
|
|
raise DriftDetectorError(f"{label} runtime commit identity is missing or ambiguous")
|
|
candidate_values = set(candidates.values())
|
|
if candidate_values != {runtime_commit}:
|
|
raise DriftDetectorError(f"{label} runtime commit candidates do not bind the observed revision")
|
|
|
|
|
|
def validate_endpoint_binding(
|
|
label: str,
|
|
route: dict[str, Any],
|
|
identity: dict[str, Any],
|
|
expected_target: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
validate_route_binding(label, route, expected_target)
|
|
if identity.get("transaction_read_only") != "on":
|
|
raise DriftDetectorError(f"{label} database transaction was not read-only")
|
|
if identity.get("database") != expected_target.get("database"):
|
|
raise DriftDetectorError(f"{label} connected database does not match the explicit target")
|
|
if identity.get("current_user") != expected_target.get("database_user"):
|
|
raise DriftDetectorError(f"{label} connected database user does not match the explicit target")
|
|
|
|
route_kind = expected_target["route_kind"]
|
|
observed_address = identity.get("server_address")
|
|
observed_port = identity.get("server_port")
|
|
expected_address = expected_target.get("database_address")
|
|
expected_port = expected_target.get("database_port")
|
|
if route_kind == "vps_docker":
|
|
if observed_address is not None or observed_port is not None:
|
|
raise DriftDetectorError(f"{label} Docker probe did not use the expected local socket endpoint")
|
|
elif route_kind == "gcp_private_cloudsql":
|
|
if not same_network_address(observed_address, expected_address) or observed_port != expected_port:
|
|
raise DriftDetectorError(f"{label} observed database address does not match the explicit endpoint")
|
|
if identity.get("ssl") is not True:
|
|
raise DriftDetectorError(f"{label} Cloud SQL probe did not use TLS")
|
|
else:
|
|
raise DriftDetectorError(f"{label} route kind is unsupported")
|
|
|
|
return {
|
|
"route_kind": route_kind,
|
|
"transport": route["transport"],
|
|
"hostname": route["hostname"],
|
|
"runtime_service": route["service"],
|
|
"database_endpoint": route["database_endpoint"],
|
|
"database": identity["database"],
|
|
"database_user": identity["current_user"],
|
|
"server_address": observed_address,
|
|
"server_port": observed_port,
|
|
"container_id": (route["database_endpoint_identity"]["container_id"] if route_kind == "vps_docker" else None),
|
|
}
|
|
|
|
|
|
REMOTE_ROUTE_INVENTORY = r"""import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
service, route_kind, transport, expected_db, database_endpoint, database_user, endpoint_runtime_identity = sys.argv[1:8]
|
|
raw = subprocess.check_output([
|
|
"systemctl", "show", service,
|
|
"-p", "ActiveState", "-p", "SubState", "-p", "MainPID", "-p", "NRestarts",
|
|
"-p", "FragmentPath", "-p", "WorkingDirectory", "--no-pager",
|
|
], text=True, stderr=subprocess.DEVNULL).strip()
|
|
state = dict(line.split("=", 1) for line in raw.splitlines() if "=" in line)
|
|
pid = int(state.get("MainPID") or 0)
|
|
if state.get("ActiveState") != "active" or state.get("SubState") != "running" or pid <= 0:
|
|
raise SystemExit("runtime service is not active/running with a positive MainPID")
|
|
cwd = Path(f"/proc/{pid}/cwd").resolve()
|
|
commit_candidates = {}
|
|
try:
|
|
value = subprocess.check_output(
|
|
["git", "-C", str(cwd), "rev-parse", "HEAD"],
|
|
text=True,
|
|
stderr=subprocess.DEVNULL,
|
|
).strip()
|
|
if re.fullmatch(r"[0-9a-f]{40}", value):
|
|
commit_candidates[f"git:{cwd}"] = value
|
|
except (FileNotFoundError, subprocess.CalledProcessError):
|
|
pass
|
|
for candidate in (
|
|
cwd / ".last-deploy-sha",
|
|
Path("/opt/teleo-eval/.last-deploy-sha"),
|
|
Path("/opt/teleo-codex/.last-deploy-sha"),
|
|
):
|
|
if candidate.is_file() and not candidate.is_symlink():
|
|
value = candidate.read_text(encoding="utf-8", errors="replace").strip()
|
|
if re.fullmatch(r"[0-9a-f]{40}", value):
|
|
commit_candidates[str(candidate)] = value
|
|
commits = sorted(set(commit_candidates.values()))
|
|
database_endpoint_identity = None
|
|
if route_kind == "vps_docker":
|
|
if not database_endpoint.startswith("docker:"):
|
|
raise SystemExit("VPS database endpoint is not a Docker selector")
|
|
container_selector = database_endpoint.removeprefix("docker:")
|
|
records = json.loads(subprocess.check_output(
|
|
["docker", "inspect", container_selector],
|
|
text=True,
|
|
stderr=subprocess.DEVNULL,
|
|
))
|
|
if not isinstance(records, list) or len(records) != 1:
|
|
raise SystemExit("VPS database endpoint did not resolve to exactly one container")
|
|
record = records[0]
|
|
container_id = record.get("Id")
|
|
container_name = str(record.get("Name") or "").removeprefix("/")
|
|
container_running = (record.get("State") or {}).get("Running") is True
|
|
if not isinstance(container_id, str) or not re.fullmatch(r"[0-9a-f]{64}", container_id):
|
|
raise SystemExit("VPS database container identity is invalid")
|
|
if (
|
|
container_selector not in {container_id, container_id[:12], container_name}
|
|
or endpoint_runtime_identity != container_id
|
|
or not container_running
|
|
):
|
|
raise SystemExit("VPS database container identity does not match the requested running endpoint")
|
|
database_endpoint_identity = {
|
|
"kind": "docker_container",
|
|
"container_id": container_id,
|
|
"container_name": container_name,
|
|
"running": container_running,
|
|
}
|
|
elif endpoint_runtime_identity != "not_applicable":
|
|
raise SystemExit("unexpected database endpoint runtime identity")
|
|
payload = {
|
|
"schema": "livingip.remoteSourceRoute.v1",
|
|
"route_kind": route_kind,
|
|
"transport": transport,
|
|
"hostname": os.uname().nodename,
|
|
"service": service,
|
|
"service_state": state,
|
|
"process_cwd": str(cwd),
|
|
"expected_database": expected_db,
|
|
"database_endpoint": database_endpoint,
|
|
"database_endpoint_identity": database_endpoint_identity,
|
|
"database_user": database_user,
|
|
"runtime_commit": commits[0] if len(commits) == 1 else None,
|
|
"commit_candidates": commit_candidates,
|
|
"commit_identity_ambiguous": len(commits) != 1,
|
|
"secret_values_emitted": False,
|
|
}
|
|
print("LEO_SOURCE_ROUTE\t" + json.dumps(payload, sort_keys=True))
|
|
"""
|
|
|
|
|
|
def ssh_base(target: str, *, identity_file: Path | None, timeout: int) -> list[str]:
|
|
command = [
|
|
"ssh",
|
|
"-T",
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
f"ConnectTimeout={timeout}",
|
|
]
|
|
if identity_file is not None:
|
|
command.extend(["-i", str(identity_file)])
|
|
command.extend([target, "--"])
|
|
return command
|
|
|
|
|
|
def remote_script(
|
|
*,
|
|
route_kind: str,
|
|
transport: str,
|
|
service: str,
|
|
database: str,
|
|
container: str | None,
|
|
database_user: str,
|
|
gcp_host: str | None,
|
|
gcp_project: str | None,
|
|
gcp_secret: str | None,
|
|
) -> str:
|
|
database_endpoint = f"docker:{container}" if route_kind == "vps_docker" else f"{gcp_host}:5432"
|
|
inventory_parts = [
|
|
"python3",
|
|
"-c",
|
|
shlex.quote(REMOTE_ROUTE_INVENTORY),
|
|
shlex.quote(service),
|
|
shlex.quote(route_kind),
|
|
shlex.quote(transport),
|
|
shlex.quote(database),
|
|
shlex.quote(database_endpoint),
|
|
shlex.quote(database_user),
|
|
]
|
|
if route_kind == "vps_docker":
|
|
assert container is not None
|
|
container_id = f"container_id=\"$(docker inspect --format '{{{{.Id}}}}' {shlex.quote(container)})\""
|
|
validate_container_id = (
|
|
'[[ "$container_id" =~ ^[0-9a-f]{64}$ ]] '
|
|
"|| { printf 'Docker container identity is invalid\\n' >&2; exit 3; }"
|
|
)
|
|
inventory = " ".join([*inventory_parts, '"$container_id"'])
|
|
query = " ".join(
|
|
[
|
|
"exec docker exec",
|
|
"-e",
|
|
shlex.quote("PGOPTIONS=-c default_transaction_read_only=on"),
|
|
"-i",
|
|
'"$container_id"',
|
|
"psql -X -Atq -v ON_ERROR_STOP=1",
|
|
"-U",
|
|
shlex.quote(database_user),
|
|
"-d",
|
|
shlex.quote(database),
|
|
]
|
|
)
|
|
return f"set -euo pipefail\n{container_id}\n{validate_container_id}\n{inventory}\n{query}\n"
|
|
assert gcp_host is not None and gcp_project is not None and gcp_secret is not None
|
|
inventory = " ".join([*inventory_parts, shlex.quote("not_applicable")])
|
|
connection = f"host={gcp_host} port=5432 dbname={database} user={database_user} sslmode=require connect_timeout=8"
|
|
return "\n".join(
|
|
[
|
|
"set -euo pipefail",
|
|
inventory,
|
|
f'password="$(gcloud secrets versions access latest --secret={shlex.quote(gcp_secret)} '
|
|
f'--project={shlex.quote(gcp_project)} --quiet)"',
|
|
"[[ -n \"$password\" ]] || { printf 'Cloud SQL credential resolved empty\\n' >&2; exit 2; }",
|
|
'export PGPASSWORD="$password"',
|
|
"export PGOPTIONS='-c default_transaction_read_only=on'",
|
|
f"exec psql {shlex.quote(connection)} -X -Atq -v ON_ERROR_STOP=1",
|
|
"",
|
|
]
|
|
)
|
|
|
|
|
|
def sanitize_error(stderr: str, *, target: str) -> str:
|
|
line = next((line.strip() for line in reversed(stderr.splitlines()) if line.strip()), "probe failed")
|
|
line = re.sub(r"(?i)(password|token|secret|credential)=\S+", r"\1=[REDACTED]", line)
|
|
line = re.sub(r"(?i)postgres(?:ql)?://\S+", "postgresql://[REDACTED]", line)
|
|
if target:
|
|
line = line.replace(target, "[TARGET]")
|
|
return line[:800]
|
|
|
|
|
|
def collect_endpoint(
|
|
*,
|
|
label: str,
|
|
command: list[str],
|
|
manifest_sql: bytes,
|
|
timeout: int,
|
|
expected_target: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
started = utc_now()
|
|
|
|
def invalid_observation(error: str, route: dict[str, Any] | None) -> dict[str, Any]:
|
|
return {
|
|
"schema": OBSERVATION_SCHEMA,
|
|
"label": label,
|
|
"status": "invalid",
|
|
"observed_at_utc": started,
|
|
"failure_class": "invalid_probe_receipt",
|
|
"error": error,
|
|
"expected_target": expected_target,
|
|
"manifest": None,
|
|
"route": route,
|
|
"cleanup": {"remote_files_created": False, "local_temp_files_created": False},
|
|
}
|
|
|
|
try:
|
|
completed = subprocess.run(
|
|
command,
|
|
input=manifest_sql,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
check=False,
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
return {
|
|
"schema": OBSERVATION_SCHEMA,
|
|
"label": label,
|
|
"status": "unreachable",
|
|
"observed_at_utc": started,
|
|
"failure_class": "probe_timeout",
|
|
"error": f"{label} probe exceeded {timeout}s",
|
|
"expected_target": expected_target,
|
|
"manifest": None,
|
|
"route": None,
|
|
"cleanup": {"remote_files_created": False, "local_temp_files_created": False},
|
|
}
|
|
stdout = completed.stdout.decode(errors="replace")
|
|
stderr = completed.stderr.decode(errors="replace")
|
|
lines = stdout.splitlines()
|
|
route_lines = [line for line in lines if line.startswith(ROUTE_PREFIX)]
|
|
route: dict[str, Any] | None = None
|
|
route_error: str | None = None
|
|
if len(route_lines) != 1:
|
|
route_error = f"{label} probe emitted {len(route_lines)} route observations"
|
|
else:
|
|
try:
|
|
candidate = json.loads(route_lines[0].removeprefix(ROUTE_PREFIX))
|
|
except json.JSONDecodeError:
|
|
route_error = f"{label} route observation is not valid JSON"
|
|
else:
|
|
if isinstance(candidate, dict):
|
|
route = candidate
|
|
else:
|
|
route_error = f"{label} route observation is not an object"
|
|
if route is not None:
|
|
try:
|
|
validate_route_binding(label, route, expected_target)
|
|
except (DriftDetectorError, KeyError, TypeError, ValueError) as exc:
|
|
route_error = str(exc)
|
|
if completed.returncode == 0 and route_error is not None:
|
|
return invalid_observation(route_error, route)
|
|
if completed.returncode != 0:
|
|
if route_error is not None and route_lines:
|
|
return invalid_observation(route_error, route)
|
|
try:
|
|
target = command[command.index("--") - 1]
|
|
except (ValueError, IndexError):
|
|
target = ""
|
|
return {
|
|
"schema": OBSERVATION_SCHEMA,
|
|
"label": label,
|
|
"status": "unreachable",
|
|
"observed_at_utc": started,
|
|
"failure_class": (
|
|
"database_probe_failed_after_route_observed"
|
|
if route is not None
|
|
else "transport_or_remote_probe_failed"
|
|
),
|
|
"exit_code": completed.returncode,
|
|
"error": sanitize_error(stderr, target=target),
|
|
"expected_target": expected_target,
|
|
"manifest": None,
|
|
"route": route,
|
|
"cleanup": {"remote_files_created": False, "local_temp_files_created": False},
|
|
}
|
|
if route is None:
|
|
return invalid_observation(route_error or f"{label} route observation is missing", None)
|
|
try:
|
|
manifest = parse_manifest_lines([line for line in lines if line not in route_lines], label)
|
|
identity = manifest["singleton"]["identity"]
|
|
fingerprint = manifest_fingerprint(manifest)
|
|
endpoint_identity = validate_endpoint_binding(label, route, identity, expected_target)
|
|
except (DriftDetectorError, KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
return invalid_observation(str(exc), route)
|
|
return {
|
|
"schema": OBSERVATION_SCHEMA,
|
|
"label": label,
|
|
"status": "observed_current",
|
|
"observed_at_utc": utc_now(),
|
|
"expected_target": expected_target,
|
|
"route": route,
|
|
"database_identity": identity,
|
|
"endpoint_identity": endpoint_identity,
|
|
"endpoint_identity_sha256": canonical_hash(endpoint_identity),
|
|
"fingerprint": fingerprint,
|
|
"manifest": manifest,
|
|
"cleanup": {"remote_files_created": False, "local_temp_files_created": False},
|
|
"secret_values_emitted": False,
|
|
}
|
|
|
|
|
|
def public_observation(observation: dict[str, Any]) -> dict[str, Any]:
|
|
return {key: value for key, value in observation.items() if key != "manifest"}
|
|
|
|
|
|
def invalid_public_observation(label: str, observation: Any, error: str) -> dict[str, Any]:
|
|
value = observation if isinstance(observation, dict) else {}
|
|
return {
|
|
"schema": OBSERVATION_SCHEMA,
|
|
"label": label,
|
|
"status": "invalid",
|
|
"observed_at_utc": value.get("observed_at_utc"),
|
|
"failure_class": "invalid_observation_payload",
|
|
"error": error,
|
|
"expected_target": value.get("expected_target") if isinstance(value.get("expected_target"), dict) else None,
|
|
"route": value.get("route") if isinstance(value.get("route"), dict) else None,
|
|
}
|
|
|
|
|
|
def observation_validation_error(
|
|
label: str,
|
|
observation: Any,
|
|
requested_target: dict[str, Any] | None = None,
|
|
) -> str | None:
|
|
if not isinstance(observation, dict):
|
|
return f"{label} observation is not an object"
|
|
if observation.get("schema") != OBSERVATION_SCHEMA:
|
|
return f"{label} observation schema is invalid"
|
|
if observation.get("label") != label:
|
|
return f"{label} observation label does not match its collector slot"
|
|
status = observation.get("status")
|
|
if status not in {"observed_current", "unreachable", "invalid"}:
|
|
return f"{label} observation status is invalid"
|
|
expected_target = observation.get("expected_target")
|
|
if not isinstance(expected_target, dict):
|
|
return f"{label} observation is missing its explicit target binding"
|
|
if requested_target is not None and expected_target != requested_target:
|
|
return f"{label} observation target does not match the requested collector target"
|
|
if status == "invalid":
|
|
return None
|
|
if status == "unreachable":
|
|
if observation.get("manifest") is not None:
|
|
return f"{label} unreachable observation contains a manifest"
|
|
if not isinstance(observation.get("failure_class"), str) or not isinstance(observation.get("error"), str):
|
|
return f"{label} unreachable observation is missing failure details"
|
|
route = observation.get("route")
|
|
if route is not None:
|
|
if not isinstance(route, dict):
|
|
return f"{label} unreachable observation route is not an object"
|
|
try:
|
|
validate_route_binding(label, route, expected_target)
|
|
except (DriftDetectorError, KeyError, TypeError, ValueError) as exc:
|
|
return str(exc)
|
|
return None
|
|
|
|
try:
|
|
route = observation["route"]
|
|
manifest = observation["manifest"]
|
|
identity = observation["database_identity"]
|
|
fingerprint = observation["fingerprint"]
|
|
endpoint_identity = observation["endpoint_identity"]
|
|
if not all(isinstance(item, dict) for item in (route, manifest, identity, fingerprint, endpoint_identity)):
|
|
raise DriftDetectorError(f"{label} current observation has a malformed object field")
|
|
manifest_identity = manifest["singleton"]["identity"]
|
|
if identity != manifest_identity:
|
|
raise DriftDetectorError(f"{label} database identity is detached from its manifest")
|
|
validate_manifest_contract(manifest, label)
|
|
computed_fingerprint = manifest_fingerprint(manifest)
|
|
if fingerprint != computed_fingerprint:
|
|
raise DriftDetectorError(f"{label} manifest fingerprint does not match its payload")
|
|
computed_endpoint_identity = validate_endpoint_binding(label, route, identity, expected_target)
|
|
if endpoint_identity != computed_endpoint_identity:
|
|
raise DriftDetectorError(f"{label} endpoint identity does not match its route and database")
|
|
if observation.get("endpoint_identity_sha256") != canonical_hash(endpoint_identity):
|
|
raise DriftDetectorError(f"{label} endpoint identity hash does not match its payload")
|
|
except (DriftDetectorError, KeyError, TypeError, ValueError, OverflowError) as exc:
|
|
return str(exc)
|
|
return None
|
|
|
|
|
|
def endpoint_collision_reason(vps: dict[str, Any], gcp: dict[str, Any]) -> str | None:
|
|
vps_identity = vps["endpoint_identity"]
|
|
gcp_identity = gcp["endpoint_identity"]
|
|
if vps["endpoint_identity_sha256"] == gcp["endpoint_identity_sha256"]:
|
|
return "VPS and GCP observations resolve to the same endpoint identity"
|
|
if vps_identity["hostname"] == gcp_identity["hostname"]:
|
|
return "VPS and GCP observations resolve to the same route hostname"
|
|
if (
|
|
vps_identity["server_address"] is not None
|
|
and same_network_address(vps_identity["server_address"], gcp_identity["server_address"])
|
|
and vps_identity["server_port"] == gcp_identity["server_port"]
|
|
and vps_identity["database"] == gcp_identity["database"]
|
|
):
|
|
return "VPS and GCP observations resolve to the same database socket"
|
|
return None
|
|
|
|
|
|
def evaluate_observations(
|
|
vps: Any,
|
|
gcp: Any,
|
|
*,
|
|
generated_at_utc: str | None = None,
|
|
requested_targets: dict[str, dict[str, Any]] | None = None,
|
|
) -> dict[str, Any]:
|
|
observations = {"vps": vps, "gcp": gcp}
|
|
public_observations: dict[str, Any] = {}
|
|
unavailable: list[str] = []
|
|
invalid: list[str] = []
|
|
validation_errors: dict[str, str] = {}
|
|
for label, observation in observations.items():
|
|
requested_target = requested_targets.get(label) if requested_targets is not None else None
|
|
error = observation_validation_error(label, observation, requested_target)
|
|
if error is not None:
|
|
invalid.append(label)
|
|
validation_errors[label] = error
|
|
public_observations[label] = invalid_public_observation(label, observation, error)
|
|
else:
|
|
public_observations[label] = public_observation(observation)
|
|
if observation["status"] == "invalid":
|
|
invalid.append(label)
|
|
elif observation["status"] == "unreachable":
|
|
unavailable.append(label)
|
|
initial_status = "invalid" if invalid else "incomplete" if unavailable else "pending_comparison"
|
|
receipt: dict[str, Any] = {
|
|
"schema": RECEIPT_SCHEMA,
|
|
"generated_at_utc": generated_at_utc or utc_now(),
|
|
"mode": "live_readonly_no_fallback",
|
|
"status": initial_status,
|
|
"comparison": None,
|
|
"observations": public_observations,
|
|
"unavailable_targets": unavailable,
|
|
"invalid_targets": invalid,
|
|
"validation_errors": validation_errors,
|
|
"fallback_used": False,
|
|
"historical_receipt_substituted": False,
|
|
"database_mutated": False,
|
|
"routing_mutated": False,
|
|
"secrets_logged": False,
|
|
"cleanup": {
|
|
"remote_files_created": False,
|
|
"local_temp_files_created": False,
|
|
"orphan_processes_expected": False,
|
|
},
|
|
}
|
|
if invalid:
|
|
receipt["decision"] = "No source-of-truth decision: at least one endpoint observation is invalid."
|
|
return receipt
|
|
if unavailable:
|
|
receipt["decision"] = "No source-of-truth decision: every target must be observed live."
|
|
return receipt
|
|
collision = endpoint_collision_reason(vps, gcp)
|
|
if collision is not None:
|
|
receipt["status"] = "invalid"
|
|
receipt["invalid_targets"] = ["vps", "gcp"]
|
|
receipt["validation_errors"] = {"endpoint_pair": collision}
|
|
receipt["decision"] = "No source-of-truth decision: endpoint independence was not proven."
|
|
return receipt
|
|
try:
|
|
problems, details = compare_manifests(
|
|
vps["manifest"],
|
|
gcp["manifest"],
|
|
max_target_query_ms=10_000.0,
|
|
max_target_source_ratio=1_000.0,
|
|
)
|
|
except (KeyError, TypeError, ValueError, OverflowError) as exc:
|
|
receipt["status"] = "invalid"
|
|
receipt["invalid_targets"] = ["vps", "gcp"]
|
|
receipt["validation_errors"] = {"manifest_comparison": str(exc)}
|
|
receipt["decision"] = "No source-of-truth decision: manifest comparison input is invalid."
|
|
return receipt
|
|
content_identical = vps["fingerprint"]["table_fingerprint_sha256"] == gcp["fingerprint"]["table_fingerprint_sha256"]
|
|
structure_identical = (
|
|
vps["fingerprint"]["structure_fingerprint_sha256"] == gcp["fingerprint"]["structure_fingerprint_sha256"]
|
|
)
|
|
runtime_revision_identical = vps["route"]["runtime_commit"] == gcp["route"]["runtime_commit"]
|
|
if not runtime_revision_identical:
|
|
problems.append("runtime_commit_mismatch")
|
|
state = (
|
|
"identical"
|
|
if not problems and content_identical and structure_identical and runtime_revision_identical
|
|
else "drift"
|
|
)
|
|
receipt["status"] = state
|
|
receipt["comparison"] = {
|
|
"state": state,
|
|
"content_identical": content_identical,
|
|
"structure_identical": structure_identical,
|
|
"runtime_revision_identical": runtime_revision_identical,
|
|
"problems": problems,
|
|
"details": details,
|
|
}
|
|
receipt["decision"] = (
|
|
"VPS and GCP manifests are identical under the reviewed parity contract."
|
|
if state == "identical"
|
|
else "Drift detected; do not promote or reroute until every listed mismatch is reconciled."
|
|
)
|
|
return receipt
|
|
|
|
|
|
def run_live(
|
|
args: argparse.Namespace,
|
|
*,
|
|
collector: Callable[..., dict[str, Any]] = collect_endpoint,
|
|
) -> tuple[dict[str, Any], int]:
|
|
manifest_sql = args.manifest.read_bytes()
|
|
manifest_sha256 = sha256_bytes(manifest_sql)
|
|
if manifest_sha256 != REVIEWED_MANIFEST_SQL_SHA256:
|
|
raise DriftDetectorError(
|
|
"manifest SQL hash does not match the reviewed parity contract: "
|
|
f"expected={REVIEWED_MANIFEST_SQL_SHA256} actual={manifest_sha256}"
|
|
)
|
|
vps_target = safe_value(args.vps_ssh_target, "--vps-ssh-target")
|
|
gcp_target = safe_value(args.gcp_ssh_target, "--gcp-ssh-target")
|
|
for value, flag in (
|
|
(args.vps_container, "--vps-container"),
|
|
(args.vps_database, "--vps-database"),
|
|
(args.vps_service, "--vps-service"),
|
|
(args.vps_database_user, "--vps-database-user"),
|
|
(args.gcp_database, "--gcp-database"),
|
|
(args.gcp_service, "--gcp-service"),
|
|
(args.gcp_database_user, "--gcp-database-user"),
|
|
(args.gcp_host, "--gcp-host"),
|
|
(args.gcp_project, "--gcp-project"),
|
|
(args.gcp_secret, "--gcp-secret"),
|
|
):
|
|
safe_value(value, flag)
|
|
if not args.vps_ssh_key.is_file():
|
|
raise DriftDetectorError("--vps-ssh-key must be an existing file")
|
|
|
|
vps_script = remote_script(
|
|
route_kind="vps_docker",
|
|
transport=vps_target,
|
|
service=args.vps_service,
|
|
database=args.vps_database,
|
|
container=args.vps_container,
|
|
database_user=args.vps_database_user,
|
|
gcp_host=None,
|
|
gcp_project=None,
|
|
gcp_secret=None,
|
|
)
|
|
gcp_script = remote_script(
|
|
route_kind="gcp_private_cloudsql",
|
|
transport=gcp_target,
|
|
service=args.gcp_service,
|
|
database=args.gcp_database,
|
|
container=None,
|
|
database_user=args.gcp_database_user,
|
|
gcp_host=args.gcp_host,
|
|
gcp_project=args.gcp_project,
|
|
gcp_secret=args.gcp_secret,
|
|
)
|
|
vps_remote_command = shlex.join(["bash", "-c", vps_script])
|
|
gcp_remote_command = shlex.join(["sudo", "-n", "bash", "-c", gcp_script])
|
|
vps_command = [
|
|
*ssh_base(vps_target, identity_file=args.vps_ssh_key.resolve(), timeout=args.connect_timeout),
|
|
vps_remote_command,
|
|
]
|
|
gcp_command = [
|
|
*ssh_base(gcp_target, identity_file=None, timeout=args.connect_timeout),
|
|
gcp_remote_command,
|
|
]
|
|
vps_expected_target = {
|
|
"route_kind": "vps_docker",
|
|
"transport": vps_target,
|
|
"runtime_service": args.vps_service,
|
|
"database_endpoint": f"docker:{args.vps_container}",
|
|
"database_address": None,
|
|
"database_port": None,
|
|
"database": args.vps_database,
|
|
"database_user": args.vps_database_user,
|
|
"identity_source": "explicit_cli_target",
|
|
}
|
|
gcp_expected_target = {
|
|
"route_kind": "gcp_private_cloudsql",
|
|
"transport": gcp_target,
|
|
"runtime_service": args.gcp_service,
|
|
"database_endpoint": f"{args.gcp_host}:5432",
|
|
"database_address": args.gcp_host,
|
|
"database_port": 5432,
|
|
"database": args.gcp_database,
|
|
"database_user": args.gcp_database_user,
|
|
"project": args.gcp_project,
|
|
"credential_source": f"secret-manager:{args.gcp_secret}",
|
|
"identity_source": "explicit_cli_target",
|
|
}
|
|
with ThreadPoolExecutor(max_workers=2, thread_name_prefix="source-drift") as pool:
|
|
vps_future = pool.submit(
|
|
collector,
|
|
label="vps",
|
|
command=vps_command,
|
|
manifest_sql=manifest_sql,
|
|
timeout=args.probe_timeout,
|
|
expected_target=vps_expected_target,
|
|
)
|
|
gcp_future = pool.submit(
|
|
collector,
|
|
label="gcp",
|
|
command=gcp_command,
|
|
manifest_sql=manifest_sql,
|
|
timeout=args.probe_timeout,
|
|
expected_target=gcp_expected_target,
|
|
)
|
|
vps = vps_future.result()
|
|
gcp = gcp_future.result()
|
|
receipt = evaluate_observations(
|
|
vps,
|
|
gcp,
|
|
requested_targets={"vps": vps_expected_target, "gcp": gcp_expected_target},
|
|
)
|
|
receipt["manifest_sql_sha256"] = manifest_sha256
|
|
return receipt, EXIT_CODE_BY_STATUS[receipt["status"]]
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output", required=True, type=Path)
|
|
parser.add_argument("--manifest", default=Path("ops/postgres_parity_manifest.sql"), type=Path)
|
|
parser.add_argument("--connect-timeout", type=int, default=12)
|
|
parser.add_argument("--probe-timeout", type=int, default=180)
|
|
parser.add_argument("--vps-ssh-target", default="root@77.42.65.182")
|
|
parser.add_argument(
|
|
"--vps-ssh-key",
|
|
default=Path.home() / ".ssh/livingip_hetzner_20260604_ed25519",
|
|
type=Path,
|
|
)
|
|
parser.add_argument("--vps-container", default="teleo-pg")
|
|
parser.add_argument("--vps-database", default="teleo")
|
|
parser.add_argument("--vps-database-user", default="postgres")
|
|
parser.add_argument("--vps-service", default="leoclean-gateway.service")
|
|
parser.add_argument("--gcp-ssh-target", default="teleo-gcp-staging")
|
|
parser.add_argument("--gcp-host", default="10.61.0.3")
|
|
parser.add_argument("--gcp-database", required=True)
|
|
parser.add_argument("--gcp-database-user", default="postgres")
|
|
parser.add_argument("--gcp-project", default="teleo-501523")
|
|
parser.add_argument("--gcp-secret", default="gcp-teleo-pgvector-standby-postgres-password")
|
|
parser.add_argument("--gcp-service", default="leoclean-gcp-prod-parallel.service")
|
|
args = parser.parse_args()
|
|
if not (1 <= args.connect_timeout <= 60):
|
|
parser.error("--connect-timeout must be between 1 and 60 seconds")
|
|
if not (10 <= args.probe_timeout <= 900):
|
|
parser.error("--probe-timeout must be between 10 and 900 seconds")
|
|
if not args.manifest.is_file():
|
|
parser.error("--manifest must be an existing file")
|
|
return args
|
|
|
|
|
|
def invalidate_current_receipt(path: Path) -> None:
|
|
"""Remove the previous canonical receipt before any new probe can start."""
|
|
try:
|
|
path.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
invalidate_current_receipt(args.output)
|
|
try:
|
|
receipt, exit_code = run_live(args)
|
|
except DriftDetectorError as exc:
|
|
receipt = {
|
|
"schema": RECEIPT_SCHEMA,
|
|
"generated_at_utc": utc_now(),
|
|
"mode": "live_readonly_no_fallback",
|
|
"status": "invalid",
|
|
"error": str(exc),
|
|
"fallback_used": False,
|
|
"historical_receipt_substituted": False,
|
|
"database_mutated": False,
|
|
"routing_mutated": False,
|
|
"secrets_logged": False,
|
|
}
|
|
exit_code = 3
|
|
write_private_json(args.output, receipt)
|
|
print_private_receipt_summary(args.output, receipt)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": receipt["status"],
|
|
"output_name": args.output.name,
|
|
"fallback_used": receipt.get("fallback_used", False),
|
|
"database_mutated": receipt.get("database_mutated", False),
|
|
"routing_mutated": receipt.get("routing_mutated", False),
|
|
},
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return exit_code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|