382 lines
13 KiB
Python
382 lines
13 KiB
Python
import argparse
|
|
import json
|
|
import stat
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from ops import run_local_canonical_postgres_rebuild as rebuild
|
|
|
|
COUNTS = {
|
|
"public.claims": 1837,
|
|
"public.sources": 4145,
|
|
"public.claim_evidence": 4670,
|
|
"public.claim_edges": 4916,
|
|
"public.reasoning_tools": 1,
|
|
"kb_stage.kb_proposals": 26,
|
|
}
|
|
|
|
|
|
def completed(command: list[str], returncode: int = 0, stdout: str = "", stderr: str = ""):
|
|
return subprocess.CompletedProcess(command, returncode, stdout, stderr)
|
|
|
|
|
|
class FakeDocker:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
restore_fails: bool = False,
|
|
preserve_on_remove: bool = False,
|
|
manifest_text: str = "",
|
|
) -> None:
|
|
self.restore_fails = restore_fails
|
|
self.preserve_on_remove = preserve_on_remove
|
|
self.manifest_text = manifest_text
|
|
self.commands: list[list[str]] = []
|
|
self.running = False
|
|
self.container_name = ""
|
|
self.image = ""
|
|
self.label = ""
|
|
self.tmpfs = ""
|
|
|
|
def __call__(self, command: list[str], *, timeout: float):
|
|
assert timeout > 0
|
|
self.commands.append(command)
|
|
|
|
if command[1:3] == ["container", "inspect"]:
|
|
if not self.running:
|
|
return completed(command, 1, stderr=f"Error: No such object: {command[-1]}")
|
|
payload = [
|
|
{
|
|
"Config": {
|
|
"Image": self.image,
|
|
"Labels": {rebuild.CANARY_LABEL: self.label},
|
|
},
|
|
"HostConfig": {
|
|
"NetworkMode": "none",
|
|
"Tmpfs": {rebuild.DATA_DIR: self.tmpfs.split(":", 1)[1]},
|
|
},
|
|
"State": {"Running": True},
|
|
}
|
|
]
|
|
return completed(command, stdout=json.dumps(payload))
|
|
|
|
if command[1] == "run":
|
|
self.running = True
|
|
self.container_name = command[command.index("--name") + 1]
|
|
self.label = command[command.index("--label") + 1].split("=", 1)[1]
|
|
self.tmpfs = command[command.index("--tmpfs") + 1]
|
|
self.image = command[-1]
|
|
return completed(command, stdout="0123456789abcdef\n")
|
|
|
|
if command[1] == "cp":
|
|
return completed(command)
|
|
|
|
if command[1] == "exec" and command[3] == "pg_restore":
|
|
if self.restore_fails:
|
|
return completed(command, 1, stderr="pg_restore: error: fixture restore failure")
|
|
return completed(command)
|
|
|
|
if command[1] == "exec" and command[3] == "psql":
|
|
if "-f" in command:
|
|
return completed(command, stdout=self.manifest_text)
|
|
sql = command[command.index("-c") + 1]
|
|
if sql == "select current_database();":
|
|
return completed(command, stdout="teleo\n")
|
|
if sql.startswith("CREATE ROLE"):
|
|
return completed(command)
|
|
if sql.startswith("select to_regclass("):
|
|
return completed(command, stdout="t\n")
|
|
if "from pg_tables" in sql:
|
|
return completed(command, stdout="47\n")
|
|
for table, count in COUNTS.items():
|
|
schema, name = table.split(".", 1)
|
|
if f'from "{schema}"."{name}"' in sql:
|
|
return completed(command, stdout=f"{count}\n")
|
|
raise AssertionError(f"unexpected psql query: {sql}")
|
|
|
|
if command[1:3] == ["rm", "--force"]:
|
|
if not self.preserve_on_remove:
|
|
self.running = False
|
|
return completed(command, stdout=f"{command[-1]}\n")
|
|
return completed(command, 1, stderr="fixture refused removal")
|
|
|
|
raise AssertionError(f"unexpected command: {command}")
|
|
|
|
|
|
def write_dump(path: Path, payload: bytes = b"fixture") -> Path:
|
|
path.write_bytes(b"PGDMP" + payload)
|
|
return path
|
|
|
|
|
|
def manifest_rows() -> list[dict]:
|
|
role = {
|
|
"name": "kb_apply",
|
|
"can_login": True,
|
|
"superuser": False,
|
|
"create_db": False,
|
|
"create_role": False,
|
|
"inherit": True,
|
|
"replication": False,
|
|
"bypass_rls": False,
|
|
}
|
|
rows = [
|
|
{
|
|
"kind": "identity",
|
|
"database": "teleo",
|
|
"server_version_num": 160010,
|
|
"transaction_read_only": "on",
|
|
"server_address": None,
|
|
"server_port": None,
|
|
"ssl": False,
|
|
"ssl_version": None,
|
|
"database_collation": "C",
|
|
"database_ctype": "C",
|
|
},
|
|
{"kind": "schemas", "items": ["kb_stage", "public"]},
|
|
{"kind": "extensions", "items": []},
|
|
{"kind": "application_roles", "items": [role]},
|
|
]
|
|
rows.extend(
|
|
{"kind": kind, "items": []}
|
|
for kind in (
|
|
"columns",
|
|
"constraints",
|
|
"indexes",
|
|
"sequences",
|
|
"views",
|
|
"functions",
|
|
"triggers",
|
|
"types",
|
|
"policies",
|
|
)
|
|
)
|
|
rows.extend(
|
|
[
|
|
{
|
|
"kind": "table",
|
|
"schema": "public",
|
|
"table": "claims",
|
|
"row_count": 1837,
|
|
"rowset_md5": "same-row-hash",
|
|
},
|
|
{
|
|
"kind": "performance",
|
|
"query": "count_claims",
|
|
"elapsed_ms": 1.0,
|
|
"observed_rows": 1837,
|
|
},
|
|
]
|
|
)
|
|
return rows
|
|
|
|
|
|
def write_manifest(path: Path) -> Path:
|
|
path.write_text("".join(json.dumps(row) + "\n" for row in manifest_rows()), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def args_for(tmp_path: Path, **overrides) -> argparse.Namespace:
|
|
values = {
|
|
"dump": write_dump(tmp_path / "canonical.dump"),
|
|
"database": "teleo",
|
|
"image": rebuild.DEFAULT_IMAGE,
|
|
"container_prefix": "teleo-canonical-rebuild-test",
|
|
"tmpfs_mb": 256,
|
|
"source_manifest": None,
|
|
"manifest_sql": Path("ops/postgres_parity_manifest.sql").resolve(),
|
|
"max_target_query_ms": 2000.0,
|
|
"max_target_source_ratio": 20.0,
|
|
"startup_timeout": 2.0,
|
|
"restore_timeout": 5.0,
|
|
"manifest_timeout": 5.0,
|
|
"command_timeout": 2.0,
|
|
"poll_interval": 0.001,
|
|
"docker_bin": "docker",
|
|
"output": tmp_path / "receipt.json",
|
|
}
|
|
values.update(overrides)
|
|
return argparse.Namespace(**values)
|
|
|
|
|
|
def test_success_uses_networkless_tmpfs_named_database_and_required_restore_flags(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
fake = FakeDocker()
|
|
monkeypatch.setattr(rebuild, "_run", fake)
|
|
|
|
result = rebuild.run_canary(args_for(tmp_path))
|
|
|
|
assert result["status"] == "pass"
|
|
assert result["container"]["name"].startswith("teleo-canonical-rebuild-test-")
|
|
assert result["container"]["named_database_psql_ready"] is True
|
|
assert result["container"]["isolation"]["network_mode"] == "none"
|
|
assert result["key_counts"] == COUNTS
|
|
assert result["canonical_schema_table_count"] == 47
|
|
assert result["cleanup"]["status"] == "pass"
|
|
assert result["cleanup"]["container_absent"] is True
|
|
|
|
run_command = next(command for command in fake.commands if command[1] == "run")
|
|
assert run_command[run_command.index("--network") + 1] == "none"
|
|
assert run_command[run_command.index("--tmpfs") + 1] == (f"{rebuild.DATA_DIR}:rw,nosuid,nodev,noexec,size=256m")
|
|
assert "--rm" in run_command
|
|
|
|
readiness_index = next(
|
|
index for index, command in enumerate(fake.commands) if "select current_database();" in command
|
|
)
|
|
restore_index = next(
|
|
index for index, command in enumerate(fake.commands) if len(command) > 3 and command[3] == "pg_restore"
|
|
)
|
|
assert readiness_index < restore_index
|
|
readiness_command = fake.commands[readiness_index]
|
|
assert readiness_command[readiness_command.index("-d") + 1] == "teleo"
|
|
assert not any("pg_isready" in part for command in fake.commands for part in command)
|
|
|
|
restore_command = fake.commands[restore_index]
|
|
assert "--no-owner" in restore_command
|
|
assert "--no-privileges" in restore_command
|
|
assert "--exit-on-error" in restore_command
|
|
|
|
|
|
def test_optional_manifest_bootstraps_allowlisted_roles_and_uses_existing_verifier(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
source_manifest = write_manifest(tmp_path / "source-manifest.jsonl")
|
|
fake = FakeDocker(manifest_text=source_manifest.read_text())
|
|
monkeypatch.setattr(rebuild, "_run", fake)
|
|
|
|
result = rebuild.run_canary(args_for(tmp_path, source_manifest=source_manifest))
|
|
|
|
assert result["status"] == "pass"
|
|
assert result["application_roles_bootstrapped"] == ["kb_apply"]
|
|
assert result["parity"]["status"] == "pass"
|
|
assert result["parity"]["verifier"] == "ops.verify_postgres_parity_manifest.compare_manifests"
|
|
assert result["parity"]["problems"] == []
|
|
assert result["parity"]["details"]["source_total_rows"] == 1837
|
|
assert result["parity"]["target_manifest"]["line_count"] == len(manifest_rows())
|
|
|
|
role_command = next(
|
|
command
|
|
for command in fake.commands
|
|
if command[1] == "exec" and command[3] == "psql" and any("CREATE ROLE" in part for part in command)
|
|
)
|
|
role_sql = role_command[role_command.index("-c") + 1]
|
|
assert 'CREATE ROLE "kb_apply" LOGIN NOSUPERUSER' in role_sql
|
|
assert "PASSWORD" not in role_sql
|
|
assert any(command[1] == "cp" and rebuild.MANIFEST_DESTINATION in command[-1] for command in fake.commands)
|
|
|
|
|
|
def test_restore_failure_still_forces_removal_and_proves_absence(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
fake = FakeDocker(restore_fails=True)
|
|
monkeypatch.setattr(rebuild, "_run", fake)
|
|
|
|
result = rebuild.run_canary(args_for(tmp_path))
|
|
|
|
assert result["status"] == "fail"
|
|
assert result["error"]["phase"] == "restore"
|
|
assert "fixture restore failure" in result["error"]["message"]
|
|
assert result["cleanup"]["force_remove_exit_code"] == 0
|
|
assert result["cleanup"]["container_absent"] is True
|
|
assert result["cleanup"]["status"] == "pass"
|
|
assert any(command[1:3] == ["rm", "--force"] for command in fake.commands)
|
|
assert fake.running is False
|
|
|
|
|
|
def test_unproven_cleanup_overrides_success_and_fails_closed(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
fake = FakeDocker(preserve_on_remove=True)
|
|
monkeypatch.setattr(rebuild, "_run", fake)
|
|
monkeypatch.setattr(rebuild.time, "sleep", lambda _seconds: None)
|
|
|
|
result = rebuild.run_canary(args_for(tmp_path))
|
|
|
|
assert result["status"] == "fail"
|
|
assert result["error"]["phase"] == "cleanup"
|
|
assert result["error"]["type"] == "CleanupProofError"
|
|
assert result["cleanup"]["container_absent"] is False
|
|
assert result["cleanup"]["status"] == "fail"
|
|
assert result["cleanup"]["absence_inspection_attempts"] == 5
|
|
|
|
|
|
def test_cleanup_interruption_is_receipted_and_does_not_bypass_absence_proof(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
fake = FakeDocker()
|
|
interrupted = False
|
|
|
|
def interrupt_cleanup(command: list[str], *, timeout: float):
|
|
nonlocal interrupted
|
|
if command[1:3] == ["rm", "--force"] and not interrupted:
|
|
interrupted = True
|
|
fake.running = False
|
|
raise rebuild.TerminationRequested("received SIGTERM")
|
|
return fake(command, timeout=timeout)
|
|
|
|
monkeypatch.setattr(rebuild, "_run", interrupt_cleanup)
|
|
|
|
result = rebuild.run_canary(args_for(tmp_path))
|
|
|
|
assert result["status"] == "fail"
|
|
assert result["error"] == {
|
|
"phase": "cleanup",
|
|
"type": "TerminationRequested",
|
|
"message": "received SIGTERM",
|
|
}
|
|
assert result["cleanup"]["container_absent"] is True
|
|
assert result["cleanup"]["status"] == "pass"
|
|
|
|
|
|
def test_invalid_dump_fails_before_start_but_still_proves_name_absent(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
invalid = tmp_path / "plain.sql"
|
|
invalid.write_text("select 1;")
|
|
fake = FakeDocker()
|
|
monkeypatch.setattr(rebuild, "_run", fake)
|
|
|
|
result = rebuild.run_canary(args_for(tmp_path, dump=invalid))
|
|
|
|
assert result["status"] == "fail"
|
|
assert result["error"]["phase"] == "validation"
|
|
assert "PGDMP" in result["error"]["message"]
|
|
assert not any(command[1] == "run" for command in fake.commands)
|
|
assert result["cleanup"]["container_absent"] is True
|
|
|
|
|
|
def test_application_role_sql_rejects_roles_outside_exact_allowlist() -> None:
|
|
source = {"singleton": {"application_roles": {"items": [{"name": "postgres"}]}}}
|
|
|
|
with pytest.raises(ValueError, match="non-allowlisted"):
|
|
rebuild.application_role_sql(source)
|
|
|
|
|
|
def test_container_names_are_unique_and_bounded() -> None:
|
|
first = rebuild.new_container_name("teleo-rebuild")
|
|
second = rebuild.new_container_name("teleo-rebuild")
|
|
|
|
assert first != second
|
|
assert first.startswith("teleo-rebuild-")
|
|
assert len(first) <= 63
|
|
|
|
|
|
def test_receipt_write_is_atomic_private_and_replaces_symlink_not_target(tmp_path: Path) -> None:
|
|
target = tmp_path / "unrelated.json"
|
|
target.write_text("keep")
|
|
receipt = tmp_path / "receipt.json"
|
|
receipt.symlink_to(target)
|
|
|
|
rebuild.write_receipt(receipt, {"status": "pass"})
|
|
|
|
assert json.loads(receipt.read_text()) == {"status": "pass"}
|
|
assert target.read_text() == "keep"
|
|
assert stat.S_IMODE(receipt.stat().st_mode) == 0o600
|