import argparse import json import stat import subprocess from pathlib import Path import pytest from ops import run_local_canonical_postgres_rebuild as rebuild COUNTS = { "public.claims": 1837, "public.sources": 4145, "public.claim_evidence": 4670, "public.claim_edges": 4916, "public.reasoning_tools": 1, "kb_stage.kb_proposals": 26, } def completed(command: list[str], returncode: int = 0, stdout: str = "", stderr: str = ""): return subprocess.CompletedProcess(command, returncode, stdout, stderr) class FakeDocker: def __init__( self, *, restore_fails: bool = False, preserve_on_remove: bool = False, manifest_text: str = "", ) -> None: self.restore_fails = restore_fails self.preserve_on_remove = preserve_on_remove self.manifest_text = manifest_text self.commands: list[list[str]] = [] self.running = False self.container_name = "" self.image = "" self.label = "" self.tmpfs = "" def __call__(self, command: list[str], *, timeout: float): assert timeout > 0 self.commands.append(command) if command[1:3] == ["container", "inspect"]: if not self.running: return completed(command, 1, stderr=f"Error: No such object: {command[-1]}") payload = [ { "Config": { "Image": self.image, "Labels": {rebuild.CANARY_LABEL: self.label}, }, "HostConfig": { "NetworkMode": "none", "Tmpfs": {rebuild.DATA_DIR: self.tmpfs.split(":", 1)[1]}, }, "State": {"Running": True}, } ] return completed(command, stdout=json.dumps(payload)) if command[1] == "run": self.running = True self.container_name = command[command.index("--name") + 1] self.label = command[command.index("--label") + 1].split("=", 1)[1] self.tmpfs = command[command.index("--tmpfs") + 1] self.image = command[-1] return completed(command, stdout="0123456789abcdef\n") if command[1] == "cp": return completed(command) if command[1] == "exec" and command[3] == "pg_restore": if self.restore_fails: return completed(command, 1, stderr="pg_restore: error: fixture restore failure") return completed(command) if command[1] == "exec" and command[3] == "psql": if "-f" in command: return completed(command, stdout=self.manifest_text) sql = command[command.index("-c") + 1] if sql == "select current_database();": return completed(command, stdout="teleo\n") if sql.startswith("CREATE ROLE"): return completed(command) if sql.startswith("select to_regclass("): return completed(command, stdout="t\n") if "from pg_tables" in sql: return completed(command, stdout="47\n") for table, count in COUNTS.items(): schema, name = table.split(".", 1) if f'from "{schema}"."{name}"' in sql: return completed(command, stdout=f"{count}\n") raise AssertionError(f"unexpected psql query: {sql}") if command[1:3] == ["rm", "--force"]: if not self.preserve_on_remove: self.running = False return completed(command, stdout=f"{command[-1]}\n") return completed(command, 1, stderr="fixture refused removal") raise AssertionError(f"unexpected command: {command}") def write_dump(path: Path, payload: bytes = b"fixture") -> Path: path.write_bytes(b"PGDMP" + payload) return path def manifest_rows() -> list[dict]: role = { "name": "kb_apply", "can_login": True, "superuser": False, "create_db": False, "create_role": False, "inherit": True, "replication": False, "bypass_rls": False, } rows = [ { "kind": "identity", "database": "teleo", "server_version_num": 160010, "transaction_read_only": "on", "server_address": None, "server_port": None, "ssl": False, "ssl_version": None, "database_collation": "C", "database_ctype": "C", }, {"kind": "schemas", "items": ["kb_stage", "public"]}, {"kind": "extensions", "items": []}, {"kind": "application_roles", "items": [role]}, ] rows.extend( {"kind": kind, "items": []} for kind in ( "columns", "constraints", "indexes", "sequences", "views", "functions", "triggers", "types", "policies", ) ) rows.extend( [ { "kind": "table", "schema": "public", "table": "claims", "row_count": 1837, "rowset_md5": "same-row-hash", }, { "kind": "performance", "query": "count_claims", "elapsed_ms": 1.0, "observed_rows": 1837, }, ] ) return rows def write_manifest(path: Path) -> Path: path.write_text("".join(json.dumps(row) + "\n" for row in manifest_rows()), encoding="utf-8") return path def args_for(tmp_path: Path, **overrides) -> argparse.Namespace: values = { "dump": write_dump(tmp_path / "canonical.dump"), "database": "teleo", "image": rebuild.DEFAULT_IMAGE, "container_prefix": "teleo-canonical-rebuild-test", "tmpfs_mb": 256, "source_manifest": None, "manifest_sql": Path("ops/postgres_parity_manifest.sql").resolve(), "max_target_query_ms": 2000.0, "max_target_source_ratio": 20.0, "startup_timeout": 2.0, "restore_timeout": 5.0, "manifest_timeout": 5.0, "command_timeout": 2.0, "poll_interval": 0.001, "docker_bin": "docker", "output": tmp_path / "receipt.json", } values.update(overrides) return argparse.Namespace(**values) def test_success_uses_networkless_tmpfs_named_database_and_required_restore_flags( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: fake = FakeDocker() monkeypatch.setattr(rebuild, "_run", fake) result = rebuild.run_canary(args_for(tmp_path)) assert result["status"] == "pass" assert result["container"]["name"].startswith("teleo-canonical-rebuild-test-") assert result["container"]["named_database_psql_ready"] is True assert result["container"]["isolation"]["network_mode"] == "none" assert result["key_counts"] == COUNTS assert result["canonical_schema_table_count"] == 47 assert result["cleanup"]["status"] == "pass" assert result["cleanup"]["container_absent"] is True run_command = next(command for command in fake.commands if command[1] == "run") assert run_command[run_command.index("--network") + 1] == "none" assert run_command[run_command.index("--tmpfs") + 1] == (f"{rebuild.DATA_DIR}:rw,nosuid,nodev,noexec,size=256m") assert "--rm" in run_command readiness_index = next( index for index, command in enumerate(fake.commands) if "select current_database();" in command ) restore_index = next( index for index, command in enumerate(fake.commands) if len(command) > 3 and command[3] == "pg_restore" ) assert readiness_index < restore_index readiness_command = fake.commands[readiness_index] assert readiness_command[readiness_command.index("-d") + 1] == "teleo" assert not any("pg_isready" in part for command in fake.commands for part in command) restore_command = fake.commands[restore_index] assert "--no-owner" in restore_command assert "--no-privileges" in restore_command assert "--exit-on-error" in restore_command def test_optional_manifest_bootstraps_allowlisted_roles_and_uses_existing_verifier( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: source_manifest = write_manifest(tmp_path / "source-manifest.jsonl") fake = FakeDocker(manifest_text=source_manifest.read_text()) monkeypatch.setattr(rebuild, "_run", fake) result = rebuild.run_canary(args_for(tmp_path, source_manifest=source_manifest)) assert result["status"] == "pass" assert result["application_roles_bootstrapped"] == ["kb_apply"] assert result["parity"]["status"] == "pass" assert result["parity"]["verifier"] == "ops.verify_postgres_parity_manifest.compare_manifests" assert result["parity"]["problems"] == [] assert result["parity"]["details"]["source_total_rows"] == 1837 assert result["parity"]["target_manifest"]["line_count"] == len(manifest_rows()) role_command = next( command for command in fake.commands if command[1] == "exec" and command[3] == "psql" and any("CREATE ROLE" in part for part in command) ) role_sql = role_command[role_command.index("-c") + 1] assert 'CREATE ROLE "kb_apply" LOGIN NOSUPERUSER' in role_sql assert "PASSWORD" not in role_sql assert any(command[1] == "cp" and rebuild.MANIFEST_DESTINATION in command[-1] for command in fake.commands) def test_restore_failure_still_forces_removal_and_proves_absence( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: fake = FakeDocker(restore_fails=True) monkeypatch.setattr(rebuild, "_run", fake) result = rebuild.run_canary(args_for(tmp_path)) assert result["status"] == "fail" assert result["error"]["phase"] == "restore" assert "fixture restore failure" in result["error"]["message"] assert result["cleanup"]["force_remove_exit_code"] == 0 assert result["cleanup"]["container_absent"] is True assert result["cleanup"]["status"] == "pass" assert any(command[1:3] == ["rm", "--force"] for command in fake.commands) assert fake.running is False def test_unproven_cleanup_overrides_success_and_fails_closed( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: fake = FakeDocker(preserve_on_remove=True) monkeypatch.setattr(rebuild, "_run", fake) monkeypatch.setattr(rebuild.time, "sleep", lambda _seconds: None) result = rebuild.run_canary(args_for(tmp_path)) assert result["status"] == "fail" assert result["error"]["phase"] == "cleanup" assert result["error"]["type"] == "CleanupProofError" assert result["cleanup"]["container_absent"] is False assert result["cleanup"]["status"] == "fail" assert result["cleanup"]["absence_inspection_attempts"] == 5 def test_cleanup_interruption_is_receipted_and_does_not_bypass_absence_proof( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: fake = FakeDocker() interrupted = False def interrupt_cleanup(command: list[str], *, timeout: float): nonlocal interrupted if command[1:3] == ["rm", "--force"] and not interrupted: interrupted = True fake.running = False raise rebuild.TerminationRequested("received SIGTERM") return fake(command, timeout=timeout) monkeypatch.setattr(rebuild, "_run", interrupt_cleanup) result = rebuild.run_canary(args_for(tmp_path)) assert result["status"] == "fail" assert result["error"] == { "phase": "cleanup", "type": "TerminationRequested", "message": "received SIGTERM", } assert result["cleanup"]["container_absent"] is True assert result["cleanup"]["status"] == "pass" def test_invalid_dump_fails_before_start_but_still_proves_name_absent( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: invalid = tmp_path / "plain.sql" invalid.write_text("select 1;") fake = FakeDocker() monkeypatch.setattr(rebuild, "_run", fake) result = rebuild.run_canary(args_for(tmp_path, dump=invalid)) assert result["status"] == "fail" assert result["error"]["phase"] == "validation" assert "PGDMP" in result["error"]["message"] assert not any(command[1] == "run" for command in fake.commands) assert result["cleanup"]["container_absent"] is True def test_application_role_sql_rejects_roles_outside_exact_allowlist() -> None: source = {"singleton": {"application_roles": {"items": [{"name": "postgres"}]}}} with pytest.raises(ValueError, match="non-allowlisted"): rebuild.application_role_sql(source) def test_container_names_are_unique_and_bounded() -> None: first = rebuild.new_container_name("teleo-rebuild") second = rebuild.new_container_name("teleo-rebuild") assert first != second assert first.startswith("teleo-rebuild-") assert len(first) <= 63 def test_receipt_write_is_atomic_private_and_replaces_symlink_not_target(tmp_path: Path) -> None: target = tmp_path / "unrelated.json" target.write_text("keep") receipt = tmp_path / "receipt.json" receipt.symlink_to(target) rebuild.write_receipt(receipt, {"status": "pass"}) assert json.loads(receipt.read_text()) == {"status": "pass"} assert target.read_text() == "keep" assert stat.S_IMODE(receipt.stat().st_mode) == 0o600