import argparse import json import stat import subprocess from pathlib import Path import pytest from ops import run_local_canonical_postgres_rebuild as rebuild COUNTS = { "public.claims": 1837, "public.sources": 4145, "public.claim_evidence": 4670, "public.claim_edges": 4916, "public.reasoning_tools": 1, "kb_stage.kb_proposals": 26, } def completed(command: list[str], returncode: int = 0, stdout: str = "", stderr: str = ""): return subprocess.CompletedProcess(command, returncode, stdout, stderr) class FakeDocker: def __init__( self, *, restore_fails: bool = False, preserve_on_remove: bool = False, manifest_text: str = "", ) -> None: self.restore_fails = restore_fails self.preserve_on_remove = preserve_on_remove self.manifest_text = manifest_text self.commands: list[list[str]] = [] self.running = False self.container_name = "" self.image = "" self.label = "" self.tmpfs = "" def __call__(self, command: list[str], *, timeout: float): assert timeout > 0 self.commands.append(command) if command[1:3] == ["container", "inspect"]: if not self.running: return completed(command, 1, stderr=f"Error: No such object: {command[-1]}") payload = [ { "Config": { "Image": self.image, "Labels": {rebuild.CANARY_LABEL: self.label}, }, "HostConfig": { "NetworkMode": "none", "Tmpfs": {rebuild.DATA_DIR: self.tmpfs.split(":", 1)[1]}, }, "State": {"Running": True}, } ] return completed(command, stdout=json.dumps(payload)) if command[1] == "run": self.running = True self.container_name = command[command.index("--name") + 1] self.label = command[command.index("--label") + 1].split("=", 1)[1] self.tmpfs = command[command.index("--tmpfs") + 1] self.image = command[-1] return completed(command, stdout="0123456789abcdef\n") if command[1] == "cp": return completed(command) if command[1] == "exec" and command[3] == "pg_restore": if self.restore_fails: return completed(command, 1, stderr="pg_restore: error: fixture restore failure") return completed(command) if command[1] == "exec" and command[3] == "psql": if "-f" in command: return completed(command, stdout=self.manifest_text) sql = command[command.index("-c") + 1] if sql == "select current_database();": return completed(command, stdout="teleo\n") if sql.startswith("CREATE ROLE"): return completed(command) if 'GRANT "kb_observatory_read" TO "kb_apply"' in sql: return completed(command) if "ON DATABASE" in sql and "$database_acl_reset$" in sql: return completed(command) if sql.startswith("select to_regclass("): return completed(command, stdout="t\n") if "from pg_tables" in sql: return completed(command, stdout="47\n") for table, count in COUNTS.items(): schema, name = table.split(".", 1) if f'from "{schema}"."{name}"' in sql: return completed(command, stdout=f"{count}\n") raise AssertionError(f"unexpected psql query: {sql}") if command[1:3] == ["rm", "--force"]: if not self.preserve_on_remove: self.running = False return completed(command, stdout=f"{command[-1]}\n") return completed(command, 1, stderr="fixture refused removal") raise AssertionError(f"unexpected command: {command}") def write_dump(path: Path, payload: bytes = b"fixture") -> Path: path.write_bytes(b"PGDMP" + payload) return path def manifest_rows() -> list[dict]: roles = [ { "name": "kb_apply", "can_login": True, "superuser": False, "create_db": False, "create_role": False, "inherit": True, "replication": False, "bypass_rls": False, "connection_limit": -1, "default_transaction_read_only": None, "settings": [], "database_settings": [{"name": "statement_timeout", "value": "17s"}], }, { "name": "kb_observatory_read", "can_login": False, "superuser": False, "create_db": False, "create_role": False, "inherit": False, "replication": False, "bypass_rls": False, "connection_limit": -1, "default_transaction_read_only": "on", "settings": [{"name": "default_transaction_read_only", "value": "on"}], "database_settings": [], }, ] rows = [ { "kind": "identity", "database": "teleo", "server_version_num": 160010, "transaction_read_only": "on", "server_address": None, "server_port": None, "ssl": False, "ssl_version": None, "database_collation": "C", "database_ctype": "C", }, {"kind": "schemas", "items": ["kb_stage", "public"]}, {"kind": "extensions", "items": []}, {"kind": "application_roles", "items": roles}, { "kind": "role_memberships", "items": [ { "role": "kb_observatory_read", "member": "kb_apply", "grantor": "postgres", "admin_option": True, "inherit_option": False, "set_option": False, } ], }, { "kind": "object_ownership", "items": [ { "object_type": "database", "schema": None, "name": "canonical_database", "identity_arguments": None, "owner": "postgres", } ], }, { "kind": "object_acl", "items": [ { "object_type": "database", "schema": None, "name": "canonical_database", "grantor": "postgres", "grantee": "PUBLIC", "privilege_type": "CONNECT", "is_grantable": False, }, *[ { "object_type": "database", "schema": None, "name": "canonical_database", "grantor": "postgres", "grantee": "postgres", "privilege_type": privilege, "is_grantable": False, } for privilege in ("CONNECT", "CREATE", "TEMPORARY") ], { "object_type": "database", "schema": None, "name": "canonical_database", "grantor": "postgres", "grantee": "kb_observatory_read", "privilege_type": "CONNECT", "is_grantable": True, }, { "object_type": "database", "schema": None, "name": "canonical_database", "grantor": "kb_observatory_read", "grantee": "kb_apply", "privilege_type": "CONNECT", "is_grantable": False, }, ], }, ] rows.extend( {"kind": kind, "items": []} for kind in ( "columns", "constraints", "indexes", "sequences", "views", "functions", "triggers", "types", "policies", ) ) rows.extend( [ { "kind": "table", "schema": "public", "table": "claims", "row_count": 1837, "rowset_md5": "c" * 32, }, { "kind": "performance", "query": "count_claims", "elapsed_ms": 1.0, "observed_rows": 1837, }, ] ) return rows def write_manifest(path: Path) -> Path: path.write_text("".join(json.dumps(row) + "\n" for row in manifest_rows()), encoding="utf-8") return path def args_for(tmp_path: Path, **overrides) -> argparse.Namespace: values = { "dump": write_dump(tmp_path / "canonical.dump"), "database": "teleo", "image": rebuild.DEFAULT_IMAGE, "container_prefix": "teleo-canonical-rebuild-test", "tmpfs_mb": 256, "source_manifest": None, "manifest_sql": Path("ops/postgres_parity_manifest.sql").resolve(), "max_target_query_ms": 2000.0, "max_target_source_ratio": 20.0, "startup_timeout": 2.0, "restore_timeout": 5.0, "manifest_timeout": 5.0, "command_timeout": 2.0, "poll_interval": 0.001, "docker_bin": "docker", "output": tmp_path / "receipt.json", } values.update(overrides) return argparse.Namespace(**values) def test_success_uses_networkless_tmpfs_named_database_and_required_restore_flags( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: fake = FakeDocker() monkeypatch.setattr(rebuild, "_run", fake) result = rebuild.run_canary(args_for(tmp_path)) assert result["status"] == "pass" assert result["container"]["name"].startswith("teleo-canonical-rebuild-test-") assert result["container"]["named_database_psql_ready"] is True assert result["container"]["isolation"]["network_mode"] == "none" assert result["key_counts"] == COUNTS assert result["canonical_schema_table_count"] == 47 assert result["cleanup"]["status"] == "pass" assert result["cleanup"]["container_absent"] is True run_command = next(command for command in fake.commands if command[1] == "run") assert run_command[run_command.index("--network") + 1] == "none" assert run_command[run_command.index("--tmpfs") + 1] == (f"{rebuild.DATA_DIR}:rw,nosuid,nodev,noexec,size=256m") assert "--rm" in run_command readiness_index = next( index for index, command in enumerate(fake.commands) if "select current_database();" in command ) restore_index = next( index for index, command in enumerate(fake.commands) if len(command) > 3 and command[3] == "pg_restore" ) assert readiness_index < restore_index readiness_command = fake.commands[readiness_index] assert readiness_command[readiness_command.index("-d") + 1] == "teleo" assert not any("pg_isready" in part for command in fake.commands for part in command) restore_command = fake.commands[restore_index] assert "--no-owner" in restore_command assert "--no-privileges" not in restore_command assert "--exit-on-error" in restore_command def test_optional_manifest_bootstraps_allowlisted_roles_and_uses_existing_verifier( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: source_manifest = write_manifest(tmp_path / "source-manifest.jsonl") fake = FakeDocker(manifest_text=source_manifest.read_text()) monkeypatch.setattr(rebuild, "_run", fake) result = rebuild.run_canary(args_for(tmp_path, source_manifest=source_manifest)) assert result["status"] == "pass" assert result["application_roles_bootstrapped"] == ["kb_apply", "kb_observatory_read"] assert result["role_setting_statement_count"] == 1 assert result["database_role_setting_statement_count"] == 1 assert result["role_membership_statement_count"] == 1 assert result["database_acl_statement_count"] == 6 assert result["parity"]["status"] == "pass" assert result["parity"]["verifier"] == "ops.verify_postgres_parity_manifest.compare_manifests" assert result["parity"]["problems"] == [] assert result["parity"]["details"]["source_total_rows"] == 1837 assert result["parity"]["target_manifest"]["line_count"] == len(manifest_rows()) role_command = next( command for command in fake.commands if command[1] == "exec" and command[3] == "psql" and any("CREATE ROLE" in part for part in command) ) role_sql = role_command[role_command.index("-c") + 1] assert 'CREATE ROLE "kb_apply" LOGIN NOSUPERUSER' in role_sql assert 'CREATE ROLE "kb_observatory_read" NOLOGIN NOSUPERUSER' in role_sql assert 'ALTER ROLE "kb_observatory_read" SET "default_transaction_read_only" TO \'on\';' in role_sql assert 'ALTER ROLE "kb_apply" IN DATABASE "teleo" SET "statement_timeout" TO \'17s\';' in role_sql assert "PASSWORD" not in role_sql membership_command = next( command for command in fake.commands if command[1] == "exec" and command[3] == "psql" and any('GRANT "kb_observatory_read" TO "kb_apply"' in part for part in command) ) membership_sql = membership_command[membership_command.index("-c") + 1] assert 'SET ROLE "postgres";' in membership_sql assert ( 'GRANT "kb_observatory_read" TO "kb_apply" WITH ADMIN TRUE, INHERIT FALSE, SET FALSE GRANTED BY "postgres";' ) in membership_sql acl_command = next( command for command in fake.commands if command[1] == "exec" and command[3] == "psql" and any("$database_acl_reset$" in part for part in command) ) acl_sql = acl_command[acl_command.index("-c") + 1] assert "REVOKE ALL PRIVILEGES ON DATABASE %I FROM PUBLIC CASCADE" in acl_sql assert 'GRANT CONNECT ON DATABASE "teleo" TO PUBLIC GRANTED BY "postgres";' in acl_sql assert ( 'GRANT CONNECT ON DATABASE "teleo" TO "kb_observatory_read" WITH GRANT OPTION GRANTED BY "postgres";' ) in acl_sql assert ( 'SET ROLE "kb_observatory_read";\n' 'GRANT CONNECT ON DATABASE "teleo" TO "kb_apply" GRANTED BY "kb_observatory_read";' ) in acl_sql assert any(command[1] == "cp" and rebuild.MANIFEST_DESTINATION in command[-1] for command in fake.commands) def test_restore_failure_still_forces_removal_and_proves_absence( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: fake = FakeDocker(restore_fails=True) monkeypatch.setattr(rebuild, "_run", fake) result = rebuild.run_canary(args_for(tmp_path)) assert result["status"] == "fail" assert result["error"]["phase"] == "restore" assert "fixture restore failure" in result["error"]["message"] assert result["cleanup"]["force_remove_exit_code"] == 0 assert result["cleanup"]["container_absent"] is True assert result["cleanup"]["status"] == "pass" assert any(command[1:3] == ["rm", "--force"] for command in fake.commands) assert fake.running is False def test_unproven_cleanup_overrides_success_and_fails_closed( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: fake = FakeDocker(preserve_on_remove=True) monkeypatch.setattr(rebuild, "_run", fake) monkeypatch.setattr(rebuild.time, "sleep", lambda _seconds: None) result = rebuild.run_canary(args_for(tmp_path)) assert result["status"] == "fail" assert result["error"]["phase"] == "cleanup" assert result["error"]["type"] == "CleanupProofError" assert result["cleanup"]["container_absent"] is False assert result["cleanup"]["status"] == "fail" assert result["cleanup"]["absence_inspection_attempts"] == 5 def test_cleanup_interruption_is_receipted_and_does_not_bypass_absence_proof( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: fake = FakeDocker() interrupted = False def interrupt_cleanup(command: list[str], *, timeout: float): nonlocal interrupted if command[1:3] == ["rm", "--force"] and not interrupted: interrupted = True fake.running = False raise rebuild.TerminationRequested("received SIGTERM") return fake(command, timeout=timeout) monkeypatch.setattr(rebuild, "_run", interrupt_cleanup) result = rebuild.run_canary(args_for(tmp_path)) assert result["status"] == "fail" assert result["error"] == { "phase": "cleanup", "type": "TerminationRequested", "message": "received SIGTERM", } assert result["cleanup"]["container_absent"] is True assert result["cleanup"]["status"] == "pass" def test_invalid_dump_fails_before_start_but_still_proves_name_absent( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: invalid = tmp_path / "plain.sql" invalid.write_text("select 1;") fake = FakeDocker() monkeypatch.setattr(rebuild, "_run", fake) result = rebuild.run_canary(args_for(tmp_path, dump=invalid)) assert result["status"] == "fail" assert result["error"]["phase"] == "validation" assert "PGDMP" in result["error"]["message"] assert not any(command[1] == "run" for command in fake.commands) assert result["cleanup"]["container_absent"] is True def test_application_role_sql_rejects_roles_outside_exact_allowlist() -> None: source = {"singleton": {"application_roles": {"items": [{"name": "postgres"}]}}} with pytest.raises(ValueError, match="non-allowlisted"): rebuild.application_role_sql(source, "teleo") def test_application_role_sql_fails_closed_without_database_scoped_settings() -> None: source = {"singleton": {row["kind"]: row for row in manifest_rows() if "kind" in row}} del source["singleton"]["application_roles"]["items"][0]["database_settings"] with pytest.raises(ValueError, match="database_settings must be a list"): rebuild.application_role_sql(source, "teleo") def test_role_membership_sql_preserves_grantor_and_all_option_bits() -> None: source = {"singleton": {row["kind"]: row for row in manifest_rows() if "kind" in row}} sql = rebuild.role_membership_sql(source) assert sql == ( 'SET ROLE "postgres";\n' 'GRANT "kb_observatory_read" TO "kb_apply" WITH ADMIN TRUE, INHERIT FALSE, SET FALSE ' 'GRANTED BY "postgres";\n' "RESET ROLE;" ) def test_database_acl_sql_resets_defaults_and_replays_public_and_grantor_chain() -> None: source = {"singleton": {row["kind"]: row for row in manifest_rows() if "kind" in row}} sql = rebuild.database_acl_sql(source, "teleo") assert "$database_acl_reset$" in sql assert 'TO PUBLIC GRANTED BY "postgres";' in sql parent = 'GRANT CONNECT ON DATABASE "teleo" TO "kb_observatory_read" WITH GRANT OPTION GRANTED BY "postgres";' child = 'GRANT CONNECT ON DATABASE "teleo" TO "kb_apply" GRANTED BY "kb_observatory_read";' assert parent in sql assert child in sql assert sql.index(parent) < sql.index(child) def test_database_acl_sql_fails_closed_on_unrepresented_grantor_dependency() -> None: source = {"singleton": {row["kind"]: row for row in manifest_rows() if "kind" in row}} child = source["singleton"]["object_acl"]["items"][-1] child["grantor"] = "unrepresented_grantor" with pytest.raises(ValueError, match="grantor dependencies cannot be replayed"): rebuild.database_acl_sql(source, "teleo") def test_database_acl_sql_rejects_unreviewed_privilege() -> None: source = {"singleton": {row["kind"]: row for row in manifest_rows() if "kind" in row}} source["singleton"]["object_acl"]["items"][0]["privilege_type"] = "DROP" with pytest.raises(ValueError, match="unsupported database privilege"): rebuild.database_acl_sql(source, "teleo") def test_container_names_are_unique_and_bounded() -> None: first = rebuild.new_container_name("teleo-rebuild") second = rebuild.new_container_name("teleo-rebuild") assert first != second assert first.startswith("teleo-rebuild-") assert len(first) <= 63 def test_receipt_write_is_atomic_private_and_replaces_symlink_not_target(tmp_path: Path) -> None: target = tmp_path / "unrelated.json" target.write_text("keep") receipt = tmp_path / "receipt.json" receipt.symlink_to(target) rebuild.write_receipt(receipt, {"status": "pass"}) assert json.loads(receipt.read_text()) == {"status": "pass"} assert target.read_text() == "keep" assert stat.S_IMODE(receipt.stat().st_mode) == 0o600