teleo-infrastructure/tests/test_v3_source_rebuild_lifecycle_postgres.py

701 lines
26 KiB
Python

from __future__ import annotations
import hashlib
import json
import secrets
import stat
import subprocess
import sys
import time
import uuid
from pathlib import Path
import pytest
from ops import run_local_corpus_knowledge_rebuild as corpus_rebuild
from ops import run_local_genesis_ledger_rebuild as genesis_rebuild
from ops import verify_teleo_v3_epistemic_contract as v3_contract
from scripts import compile_kb_source_packet as compiler
from tests import test_run_local_genesis_ledger_rebuild as genesis_test
from tests import test_v3_apply_worker_postgres as worker_test
REPO_ROOT = Path(__file__).resolve().parents[1]
CORPUS_REBUILD = REPO_ROOT / "ops" / "run_local_corpus_knowledge_rebuild.py"
SOURCE_COMPILER = REPO_ROOT / "scripts" / "compile_kb_source_packet.py"
PROPOSAL_STAGER = REPO_ROOT / "scripts" / "stage_normalized_proposal.py"
REVIEW_PACKET = REPO_ROOT / "scripts" / "kb_proposal_review_packet.py"
APPROVER = REPO_ROOT / "scripts" / "approve_proposal.py"
APPLY_WORKER = REPO_ROOT / "scripts" / "apply_worker.py"
MATERIAL_EXPORTER = REPO_ROOT / "scripts" / "export_kb_transition_replay_material.py"
LEDGER_BUILDER = REPO_ROOT / "ops" / "build_v3_genesis_ledger_bundle.py"
GENESIS_REBUILDER = REPO_ROOT / "ops" / "run_local_genesis_ledger_rebuild.py"
SOURCE_CONTAINER_LABEL = "livingip.test.v3-source-rebuild-lifecycle"
STAGING_LABEL = "com.livingip.leo.checkpoint=disposable"
DATABASE = "teleo_v3_source_lifecycle"
SYNTHETIC_REVIEW_NOTE = "SYNTHETIC TEST ONLY: operator-supplied fixture approval; this is not human review proof."
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _run(command: list[str], *, timeout: int = 240) -> subprocess.CompletedProcess[str]:
completed = subprocess.run(
command,
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=False,
timeout=timeout,
)
assert completed.returncode == 0, (
f"command failed ({completed.returncode}): {command[0]} {Path(command[1]).name if len(command) > 1 else ''}\n"
f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
)
return completed
def _docker(*arguments: str, check: bool = True, timeout: int = 120) -> subprocess.CompletedProcess[str]:
completed = subprocess.run(
["docker", *arguments],
text=True,
capture_output=True,
check=False,
timeout=timeout,
)
if check:
assert completed.returncode == 0, completed.stderr or completed.stdout
return completed
def _write_private(path: Path, content: str | bytes) -> Path:
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
path.parent.chmod(0o700)
if isinstance(content, bytes):
path.write_bytes(content)
else:
path.write_text(content, encoding="utf-8")
path.chmod(0o600)
return path
def _start_postgres(name: str, database: str, run_label: str, *, staging_target: bool) -> str:
command = [
"run",
"--detach",
"--rm",
"--network",
"none",
"--name",
name,
"--label",
f"{SOURCE_CONTAINER_LABEL}={run_label}",
"--tmpfs",
"/var/lib/postgresql/data:rw,nosuid,nodev,noexec,size=384m",
"--env",
f"POSTGRES_DB={database}",
"--env",
"POSTGRES_HOST_AUTH_METHOD=trust",
]
if staging_target:
command.extend(("--label", STAGING_LABEL))
command.append(v3_contract.POSTGRES_IMAGE)
container_id = _docker(*command).stdout.strip()
assert container_id
for _ in range(180):
ready = _docker(
"exec",
name,
"pg_isready",
"-U",
"postgres",
"-h",
"127.0.0.1",
"-d",
database,
check=False,
timeout=10,
)
if ready.returncode == 0:
break
time.sleep(0.25)
else:
pytest.fail(f"disposable PostgreSQL did not become ready: {name}")
inspected = json.loads(_docker("inspect", name).stdout)[0]
assert inspected["HostConfig"]["NetworkMode"] == "none"
assert inspected["NetworkSettings"]["Ports"] == {}
assert inspected["Mounts"] == []
assert "/var/lib/postgresql/data" in inspected["HostConfig"]["Tmpfs"]
assert inspected["Config"]["Labels"][SOURCE_CONTAINER_LABEL] == run_label
if staging_target:
assert inspected["Config"]["Labels"]["com.livingip.leo.checkpoint"] == "disposable"
return container_id
def _remove_container(name: str) -> None:
_docker("rm", "--force", name, check=False)
def _assert_container_absent(name: str) -> None:
assert _docker("inspect", name, check=False).returncode != 0
names = _docker(
"container",
"ls",
"--all",
"--filter",
f"name=^/{name}$",
"--format",
"{{.Names}}",
)
assert names.stdout.strip() == ""
def _capture_dump(container: str, database: str, destination: Path) -> None:
remote = f"/tmp/{destination.name}"
_docker(
"exec",
container,
"pg_dump",
"-U",
"postgres",
"-d",
database,
"--format=custom",
f"--file={remote}",
)
_docker("cp", f"{container}:{remote}", str(destination))
destination.chmod(0o600)
def _capture_manifest(container: str, database: str, destination: Path) -> None:
genesis_test.capture_manifest(container, database, destination)
destination.chmod(0o600)
def _json_stdout(completed: subprocess.CompletedProcess[str]) -> dict:
value = json.loads(completed.stdout)
assert isinstance(value, dict)
return value
def _assert_private_files(root: Path) -> None:
files = [path for path in root.rglob("*") if path.is_file()]
assert files
for path in files:
assert stat.S_IMODE(path.stat().st_mode) == 0o600, path
@pytest.mark.skipif(not v3_contract.docker_available(), reason="Docker daemon is required")
def test_v3_source_rebuild_lifecycle_is_exact_networkless_and_cleanup_proven(tmp_path: Path) -> None:
if _docker("image", "inspect", v3_contract.POSTGRES_IMAGE, check=False).returncode != 0:
pytest.skip("the pinned PostgreSQL image must already exist; this test never pulls over the network")
run_id = uuid.uuid4().hex[:12]
source_container = f"teleo-v3-source-{run_id}"
rebuild_prefix = f"teleo-v3-rebuild-{run_id[:8]}"
private_root = tmp_path / "private"
private_root.mkdir(mode=0o700)
password = secrets.token_hex(24)
source_started = False
rebuild_container: str | None = None
try:
corpus_root = private_root / "corpus"
source_text = (
"# Canonical review boundary\n\n"
"A staged knowledge proposal remains non-canonical until explicit review and guarded apply both succeed.\n"
)
source_path = _write_private(corpus_root / "review-boundary.md", source_text)
inventory = private_root / "source-inventory"
built_inventory = _run(
[
sys.executable,
str(CORPUS_REBUILD),
"--source-root",
str(corpus_root),
"--output",
str(inventory),
"--mode",
"source-inventory-only",
]
)
inventory_status = _json_stdout(built_inventory)
assert inventory_status["status"] == "pass"
assert inventory_status["source_count"] == 1
verified_inventory = _run(
[
sys.executable,
str(CORPUS_REBUILD),
"--verify-bundle",
str(inventory),
]
)
assert _json_stdout(verified_inventory)["status"] == "pass"
source_manifest = json.loads((inventory / corpus_rebuild.SOURCE_MANIFEST_PATH).read_text(encoding="utf-8"))
assert len(source_manifest["sources"]) == 1
source_row = source_manifest["sources"][0]
snapshot = inventory / source_row["snapshot_path"]
assert snapshot.read_bytes() == source_path.read_bytes()
assert source_row["artifact_sha256"] == _sha256(snapshot)
assert snapshot.name == f"{source_row['artifact_sha256']}.utf8"
quote = (
"A staged knowledge proposal remains non-canonical until explicit review and guarded apply both succeed."
)
text_bytes = snapshot.read_bytes()
quote_bytes = quote.encode("utf-8")
quote_start = text_bytes.index(quote_bytes)
artifact_sha256 = _sha256(snapshot)
extractor_spec_bytes = (
b"livingip synthetic lifecycle extractor specification\n"
b"name=synthetic-v3-source-lifecycle\n"
b"version=1.0.0\n"
b"quotes=exact-utf8-byte-locators\n"
b"approval=test-operator-supplied-not-human-proof\n"
)
extractor_spec = _write_private(private_root / "extractor.spec", extractor_spec_bytes)
extraction_manifest = {
"schema": compiler.MANIFEST_SCHEMA_V3,
"artifact_sha256": artifact_sha256,
"extracted_text_sha256": artifact_sha256,
"extractor": {
"name": "synthetic-v3-source-lifecycle",
"version": "1.0.0",
"spec_sha256": hashlib.sha256(extractor_spec_bytes).hexdigest(),
},
"agent_id": v3_contract.AGENT_ID,
"source": {
"identity": f"document:sha256:{artifact_sha256}",
"source_key": "canonical_review_boundary",
"source_type": "article",
"title": "Canonical review boundary",
"locator": f"artifact://sha256/{artifact_sha256}",
"access_scope": "collective_internal",
"ingestion_origin": "operator_upload",
"provenance_status": "verified",
"source_class": "primary_record",
"auto_promotion_policy": "human_required",
"captured_at": "2026-07-19T00:00:00+00:00",
},
"claims": [
{
"claim_key": "review_and_apply_precede_canonical_state",
"type": "empirical",
"proposition": (
"A staged knowledge proposal is not canonical until explicit review and guarded apply succeed."
),
"body_markdown": "The exact retained source states both gates and their order.",
"scope": "The exact content-addressed source snapshot used by this disposable lifecycle.",
"access_scope": "collective_internal",
"owner_agent_id": None,
"status": "active",
"revision_criteria": "Revise if the exact source bytes or guarded apply contract changes.",
"revision_kind": "original",
"supersedes_id": None,
"tags": ["genesis", "provenance", "review"],
"evidence": [
{
"evidence_key": "exact_review_boundary_sentence",
"quote": quote,
"polarity": "supports",
"locator": {
"unit": "utf8_bytes",
"start": quote_start,
"end": quote_start + len(quote_bytes),
},
"rationale": "The exact byte-bound sentence directly supports the bounded proposition.",
}
],
"assessment": {
"support_tier": "strong",
"challenge_tier": "none",
"overall_state": "support_dominant",
"rationale": "One exact verified primary-record sentence directly supports the claim.",
"supersedes_assessment_id": None,
},
}
],
}
manifest_path = _write_private(
private_root / "synthetic-extraction-manifest.json",
json.dumps(extraction_manifest, indent=2, sort_keys=True) + "\n",
)
compiled_bundle_path = private_root / "compiled-v3-bundle.json"
compiled = _run(
[
sys.executable,
str(SOURCE_COMPILER),
"--artifact",
str(snapshot),
"--text",
str(snapshot),
"--manifest",
str(manifest_path),
"--extractor-spec",
str(extractor_spec),
"--output",
str(compiled_bundle_path),
]
)
assert _json_stdout(compiled)["status"] == "pending_review"
compiled_bundle = json.loads(compiled_bundle_path.read_text(encoding="utf-8"))
child = compiled_bundle["strict_child_proposal"]
proposal_id = child["id"]
apply_payload = child["payload"]["apply_payload"]
assert compiled_bundle["hashes"]["artifact_sha256"] == artifact_sha256
assert apply_payload["sources"][0]["content_hash"] == artifact_sha256
source_container_id = _start_postgres(source_container, DATABASE, run_id, staging_target=True)
source_started = True
genesis_test.docker_psql(source_container, DATABASE, v3_contract.BOOTSTRAP_SQL)
genesis_test.docker_psql(source_container, DATABASE, worker_test._supplemental_bootstrap())
genesis_test.docker_psql(
source_container,
DATABASE,
(f"alter role kb_review login password '{password}';alter role kb_apply login password '{password}';"),
)
genesis_test.docker_psql(
source_container,
DATABASE,
worker_test.PREREQUISITES.read_text(encoding="utf-8"),
)
genesis_dump = private_root / "baseline-genesis.dump"
genesis_manifest = private_root / "baseline-genesis-manifest.jsonl"
_capture_dump(source_container, DATABASE, genesis_dump)
_capture_manifest(source_container, DATABASE, genesis_manifest)
genesis_test.docker_psql(source_container, DATABASE, "alter role kb_apply nologin;")
genesis_test.docker_psql(
source_container,
DATABASE,
worker_test.MIGRATION.read_text(encoding="utf-8"),
)
genesis_test.docker_psql(
source_container,
DATABASE,
worker_test.PREREQUISITES.read_text(encoding="utf-8"),
)
secrets_file = _write_private(
private_root / "database-secrets.env",
f"KB_APPLY_DB_PASSWORD={password}\nKB_REVIEW_DB_PASSWORD={password}\n",
)
staged = _run(
[
sys.executable,
str(PROPOSAL_STAGER),
"--container",
source_container,
"--db",
DATABASE,
"--compiled-v3-bundle",
str(compiled_bundle_path),
]
)
stage_status = _json_stdout(staged)
assert stage_status == {
"bound_container_id": source_container_id,
"contract_version": 3,
"created": True,
"payload_sha256": child["payload_sha256"],
"proposal_id": proposal_id,
"status": "pending_review",
}
reviewed = _run(
[
sys.executable,
str(REVIEW_PACKET),
"--proposal-id",
proposal_id,
"--format",
"json",
"--secrets-file",
str(secrets_file),
"--container",
source_container,
"--db",
DATABASE,
"--host",
"127.0.0.1",
"--role",
"kb_review",
]
)
packets = _json_stdout(reviewed)["packets"]
assert len(packets) == 1
packet = packets[0]
assert packet["proposal_id"] == proposal_id
assert packet["review_state"] == "needs_human_review"
assert packet["strict_review_material"]["rows"] == {
key: apply_payload[key]
for key in ("claims", "sources", "evidence", "edges", "assessments", "reasoning_tools")
}
assert packet["strict_review_material"]["normalization_manifest"] == apply_payload["normalization_manifest"]
approved = _run(
[
sys.executable,
str(APPROVER),
proposal_id,
"--reviewed-by",
"m3taversal",
"--review-note",
SYNTHETIC_REVIEW_NOTE,
"--secrets-file",
str(secrets_file),
"--container",
source_container,
"--db",
DATABASE,
"--host",
"127.0.0.1",
"--role",
"kb_review",
]
)
approval_status = _json_stdout(approved)
assert approval_status["id"] == proposal_id
assert approval_status["status"] == "approved"
assert approval_status["reviewed_by_handle"] == "m3taversal"
receipts = private_root / "apply-receipts"
failure_state = private_root / "apply-worker-state.json"
applied = _run(
[
sys.executable,
str(APPLY_WORKER),
"--enable",
"--contract-version",
"3",
"--proposal-id",
proposal_id,
"--max-per-tick",
"1",
"--secrets-file",
str(secrets_file),
"--failure-state-file",
str(failure_state),
"--receipt-dir",
str(receipts),
"--container",
source_container,
"--db",
DATABASE,
"--host",
"127.0.0.1",
"--role",
"kb_apply",
]
)
assert f"applied {proposal_id} (approve_claim)" in applied.stdout
assert json.loads(failure_state.read_text(encoding="utf-8")) == {}
failure_state.unlink()
replay_receipt = receipts / f"{proposal_id}.json"
receipt = json.loads(replay_receipt.read_text(encoding="utf-8"))
assert receipt["proposal"]["reviewed_by_handle"] == "m3taversal"
assert receipt["canonical_rows"]["sources"][0]["content_hash"] == artifact_sha256
assert receipt["canonical_rows"]["claims"][0]["proposition"] == extraction_manifest["claims"][0]["proposition"]
material_path = private_root / "entry-0001.json"
exported = _run(
[
sys.executable,
str(MATERIAL_EXPORTER),
proposal_id,
"--replay-receipt",
str(replay_receipt),
"--sequence",
"1",
"--output",
str(material_path),
"--secrets-file",
str(secrets_file),
"--container",
source_container,
"--db",
DATABASE,
"--host",
"127.0.0.1",
"--role",
"kb_apply",
]
)
assert exported.stdout.strip() == f"replay material created: proposal={proposal_id} sequence=1"
material = json.loads(material_path.read_text(encoding="utf-8"))
assert material["contract_version"] == genesis_rebuild.MATERIAL_CONTRACT_VERSION
assert material["applied_proposal"]["payload"]["apply_payload"]["contract_version"] == 3
assert material["approved_proposal"]["review_note"] == SYNTHETIC_REVIEW_NOTE
assert material["approval_snapshot"]["reviewed_by_db_role"] == "kb_review"
final_manifest = private_root / "final-parity-manifest.jsonl"
_capture_manifest(source_container, DATABASE, final_manifest)
_remove_container(source_container)
source_started = False
_assert_container_absent(source_container)
labeled_sources = _docker(
"container",
"ls",
"--all",
"--filter",
f"label={SOURCE_CONTAINER_LABEL}={run_id}",
"--format",
"{{.Names}}",
)
assert labeled_sources.stdout.strip() == ""
review_note_file = _write_private(private_root / "admission-review-note.txt", SYNTHETIC_REVIEW_NOTE + "\n")
ledger_path = private_root / "v3-genesis-ledger.json"
admission_path = private_root / "v3-admission-manifest.json"
built_ledger = _run(
[
sys.executable,
str(LEDGER_BUILDER),
"--genesis-dump",
str(genesis_dump),
"--genesis-manifest",
str(genesis_manifest),
"--final-manifest",
str(final_manifest),
"--material",
str(material_path),
"--output-ledger",
str(ledger_path),
"--emit-admission-manifest",
"--admission-output",
str(admission_path),
"--reviewed-by",
"m3taversal",
"--reviewed-by-agent-id",
worker_test.REVIEWER_AGENT_ID,
"--reviewed-at",
"2026-07-19T00:00:00+00:00",
"--review-note-file",
str(review_note_file),
"--admission-scope",
"knowledge_commons",
"--admission-basis",
"reviewed_primary_source",
]
)
ledger_status = _json_stdout(built_ledger)
assert ledger_status["status"] == "pass"
assert ledger_status["action"] == "build"
ledger_sha256 = _sha256(ledger_path)
admission_sha256 = _sha256(admission_path)
admission = json.loads(admission_path.read_text(encoding="utf-8"))
assert admission["reviewed_by_handle"] == "m3taversal"
assert admission["review_note"] == SYNTHETIC_REVIEW_NOTE + "\n"
verified_ledger = _run(
[
sys.executable,
str(LEDGER_BUILDER),
"--genesis-dump",
str(genesis_dump),
"--genesis-manifest",
str(genesis_manifest),
"--verify-ledger",
str(ledger_path),
"--ledger-sha256",
ledger_sha256,
"--admission-manifest",
str(admission_path),
"--admission-manifest-sha256",
admission_sha256,
]
)
verify_status = _json_stdout(verified_ledger)
assert verify_status["status"] == "pass"
assert verify_status["action"] == "verify"
reconstruction_receipt = private_root / "reconstruction-receipt.json"
rebuilt = _run(
[
sys.executable,
str(GENESIS_REBUILDER),
"--genesis-dump",
str(genesis_dump),
"--genesis-manifest",
str(genesis_manifest),
"--ledger",
str(ledger_path),
"--ledger-sha256",
ledger_sha256,
"--admission-manifest",
str(admission_path),
"--admission-manifest-sha256",
admission_sha256,
"--database",
DATABASE,
"--container-prefix",
rebuild_prefix,
"--tmpfs-mb",
"384",
"--max-target-query-ms",
"5000",
"--max-target-source-ratio",
"100",
"--output",
str(reconstruction_receipt),
],
timeout=360,
)
assert _json_stdout(rebuilt)["status"] == "pass"
reconstruction = json.loads(reconstruction_receipt.read_text(encoding="utf-8"))
rebuild_container = reconstruction["container"]["name"]
assert reconstruction["status"] == "pass"
assert reconstruction["genesis_parity"]["status"] == "pass"
assert reconstruction["genesis_parity"]["problems"] == []
assert reconstruction["v3_contract"]["status"] == "pass"
assert reconstruction["ledger"]["entries"][0]["proposal_exact"] is True
assert reconstruction["ledger"]["entries"][0]["canonical_rows_exact"] is True
assert reconstruction["admission"]["status"] == "review_metadata_supplied"
assert reconstruction["admission"]["canonical_retrieval_eligible"] is False
assert reconstruction["admission"]["cryptographically_authenticated"] is False
assert reconstruction["final_parity"]["status"] == "pass"
assert reconstruction["final_parity"]["problems"] == []
assert reconstruction["cleanup"]["container_absent"] is True
assert reconstruction["safety"] == {
"network_access_available_to_container": False,
"persistent_postgres_storage_used": False,
"private_replay_material_emitted": False,
"production_database_touched": False,
"vps_gcp_telegram_or_live_db_used": False,
}
_assert_container_absent(rebuild_container)
rebuilt_labels = _docker(
"container",
"ls",
"--all",
"--filter",
f"label={genesis_rebuild.canonical_rebuild.CANARY_LABEL}={rebuild_container}",
"--format",
"{{.Names}}",
)
assert rebuilt_labels.stdout.strip() == ""
_assert_private_files(private_root)
finally:
if source_started:
_remove_container(source_container)
_assert_container_absent(source_container)
if rebuild_container is not None:
_assert_container_absent(rebuild_container)
orphaned = _docker(
"container",
"ls",
"--all",
"--filter",
f"label={SOURCE_CONTAINER_LABEL}={run_id}",
"--format",
"{{.Names}}",
)
assert orphaned.stdout.strip() == ""