diff --git a/docs/gcp-leoclean-nosend-preflight.md b/docs/gcp-leoclean-nosend-preflight.md index 36ac724..a82622b 100644 --- a/docs/gcp-leoclean-nosend-preflight.md +++ b/docs/gcp-leoclean-nosend-preflight.md @@ -12,7 +12,11 @@ architecture, private-only VM attachment to `teleo-staging-europe-west6` (`10.60.0.0/20`), Private Google Access, immutable Docker Artifact Registry repository, exact private PostgreSQL 16 Cloud SQL endpoint, host architecture, Docker server, Docker storage floor, unused service/container -names, and private TCP/5432 reachability from the VM. +names, and private TCP/5432 reachability from the VM. Before the first Docker +client command, it requires `docker.service` to be loaded, active, and running; +after the final client command it requires unchanged service state, `MainPID`, +and restart count. This prevents a read probe from activating Docker through +`docker.socket` or silently accepting a daemon restart. ## Authority boundary @@ -42,10 +46,13 @@ python3 ops/check_gcp_leoclean_nosend_preflight.py \ The output parent must already exist, be owned by the operator, and have no group/other write permission. The output name must not exist. The checker creates it once with no-clobber/no-symlink semantics, keeps the same file and -directory identity open through publication, writes mode `0600`, and fsyncs -the file and directory. It never removes or overwrites the output pathname; -an ambiguous publication failure can therefore leave a private residue for -operator inspection. +directory identity open through publication, writes and fsyncs a mode-`0600` +private staged inode, and only then hard-links that complete inode to the +canonical output name without replacement. The retained staged and canonical +names are two links to the same inode; they are never removed automatically. +A pre-publication failure therefore leaves only the private staged residue, +not a valid-looking canonical pass receipt. A later ambiguous publication +failure preserves the exact retained inode for operator inspection. Authentication, permission, connection, and receipt-publication failures emit only a bounded category; command stderr and filesystem exception details are diff --git a/docs/workstreams/gcp-leoclean/goals.md b/docs/workstreams/gcp-leoclean/goals.md index b5afa94..5d5876d 100644 --- a/docs/workstreams/gcp-leoclean/goals.md +++ b/docs/workstreams/gcp-leoclean/goals.md @@ -73,19 +73,22 @@ cut over safely and retire the VPS after soak. bounded IAP/SSH path uses a fixed remote command and a maximum five-minute key lifetime. Receipt publication now requires a pre-existing private operator-owned directory, creates one mode-`0600` no-clobber/no-symlink - file through a held directory descriptor, verifies stable identity, fsyncs - both file and directory, and preserves ambiguous residues. No live run is - currently authorized. + staged inode through a held directory descriptor, fsyncs it before a + no-clobber canonical hard link, and retains both links without automatic + cleanup. The host probe proves Docker loaded/active/running before its first + client call and unchanged service state, `MainPID`, and restart count after + the last, preventing socket activation from changing runtime state. No live + run is currently authorized. - Artifact Registry cleanup policy is not an additional hard gate for the package: immutable tags prevent deletion of the retained tagged candidate. The post-push/finalization path must still re-prove the exact retained candidate tag-to-digest binding immediately before deployment. -- Current offline evidence for the preflight extraction: 243 focused +- Current offline evidence for the preflight extraction: 250 focused preflight/package/OCI tests pass; Ruff, Python compilation, formatting, and - diff checks pass. The repository suite produced 2,497 passes and 5 expected + diff checks pass. The repository suite produced 2,504 passes and 5 expected skips inside the desktop sandbox; all 18 sandbox-denied Docker, loopback, and Swift-cache cases passed in a separately authorized 59-test local rerun, - reconciling the suite to 2,515 passes and 5 skips. No cloud or registry + reconciling the suite to 2,522 passes and 5 skips. No cloud or registry endpoint was contacted. - Historical only: the superseded preflight branch records an earlier live observation of `e2-standard-4` on `teleo-staging-europe-west6` diff --git a/ops/check_gcp_leoclean_nosend_preflight.py b/ops/check_gcp_leoclean_nosend_preflight.py index 1aeb62f..8b7a48f 100644 --- a/ops/check_gcp_leoclean_nosend_preflight.py +++ b/ops/check_gcp_leoclean_nosend_preflight.py @@ -14,6 +14,7 @@ import shlex import stat import subprocess import sys +import uuid from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path @@ -397,6 +398,16 @@ project=\"$(metadata project/project-id)\" instance=\"$(metadata instance/name)\" service_account=\"$(metadata instance/service-accounts/default/email)\" test -x /usr/bin/docker +docker_service_load_state_before=\"$(systemctl show docker.service --property=LoadState --value)\" +docker_service_active_state_before=\"$(systemctl show docker.service --property=ActiveState --value)\" +docker_service_sub_state_before=\"$(systemctl show docker.service --property=SubState --value)\" +docker_service_main_pid_before=\"$(systemctl show docker.service --property=MainPID --value)\" +docker_service_n_restarts_before=\"$(systemctl show docker.service --property=NRestarts --value)\" +test \"$docker_service_load_state_before\" = loaded +test \"$docker_service_active_state_before\" = active +test \"$docker_service_sub_state_before\" = running +test \"$docker_service_main_pid_before\" -gt 1 +test \"$docker_service_n_restarts_before\" -ge 0 docker_server_version=\"$(sudo -n /usr/bin/docker version --format '{{{{.Server.Version}}}}')\" docker_server_os=\"$(sudo -n /usr/bin/docker version --format '{{{{.Server.Os}}}}')\" docker_server_arch=\"$(sudo -n /usr/bin/docker version --format '{{{{.Server.Arch}}}}')\" @@ -404,11 +415,26 @@ docker_root=\"$(sudo -n /usr/bin/docker info --format '{{{{.DockerRootDir}}}}')\ docker_root_free_bytes=\"$(sudo -n /usr/bin/df -B1 --output=avail \"$docker_root\" | tail -n 1 | tr -d ' ')\" unit_load_state=\"$(systemctl show {shlex.quote(package.SERVICE)} --property=LoadState --value)\" container_collision_count=\"$(sudo -n /usr/bin/docker ps -aq --filter 'name=^/{package.CONTAINER_NAME}$' | wc -l | tr -d ' ')\" +docker_service_load_state_after=\"$(systemctl show docker.service --property=LoadState --value)\" +docker_service_active_state_after=\"$(systemctl show docker.service --property=ActiveState --value)\" +docker_service_sub_state_after=\"$(systemctl show docker.service --property=SubState --value)\" +docker_service_main_pid_after=\"$(systemctl show docker.service --property=MainPID --value)\" +docker_service_n_restarts_after=\"$(systemctl show docker.service --property=NRestarts --value)\" +test \"$docker_service_load_state_after\" = \"$docker_service_load_state_before\" +test \"$docker_service_active_state_after\" = \"$docker_service_active_state_before\" +test \"$docker_service_sub_state_after\" = \"$docker_service_sub_state_before\" +test \"$docker_service_main_pid_after\" = \"$docker_service_main_pid_before\" +test \"$docker_service_n_restarts_after\" = \"$docker_service_n_restarts_before\" if timeout 5 bash -c 'exec 3<>/dev/tcp/{private_ip}/5432; exec 3<&-; exec 3>&-'; then cloudsql_tcp=reachable; else cloudsql_tcp=unreachable; fi printf 'architecture=%s\n' \"$architecture\" printf 'project=%s\n' \"$project\" printf 'instance=%s\n' \"$instance\" printf 'service_account=%s\n' \"$service_account\" +printf 'docker_service_load_state=%s\n' \"$docker_service_load_state_after\" +printf 'docker_service_active_state=%s\n' \"$docker_service_active_state_after\" +printf 'docker_service_sub_state=%s\n' \"$docker_service_sub_state_after\" +printf 'docker_service_main_pid=%s\n' \"$docker_service_main_pid_after\" +printf 'docker_service_n_restarts=%s\n' \"$docker_service_n_restarts_after\" printf 'docker_server_version=%s\n' \"$docker_server_version\" printf 'docker_server_os=%s\n' \"$docker_server_os\" printf 'docker_server_arch=%s\n' \"$docker_server_arch\" @@ -448,6 +474,11 @@ def _parse_host_output(output: str) -> dict[str, str]: "project", "instance", "service_account", + "docker_service_load_state", + "docker_service_active_state", + "docker_service_sub_state", + "docker_service_main_pid", + "docker_service_n_restarts", "docker_server_version", "docker_server_os", "docker_server_arch", @@ -477,8 +508,15 @@ def _host_readback(runner: Runner, private_ip: str) -> dict[str, Any]: try: free_bytes = int(value["docker_root_free_bytes"]) collision_count = int(value["container_collision_count"]) + docker_main_pid = int(value["docker_service_main_pid"]) + docker_n_restarts = int(value["docker_service_n_restarts"]) except ValueError as exc: raise PreflightError("invalid_readback", "host numeric field was invalid") from exc + _require(value["docker_service_load_state"] == "loaded", "Docker service is not loaded") + _require(value["docker_service_active_state"] == "active", "Docker service is not active") + _require(value["docker_service_sub_state"] == "running", "Docker service is not running") + _require(docker_main_pid > 1, "Docker service MainPID is invalid") + _require(docker_n_restarts >= 0, "Docker service restart count is invalid") _require(free_bytes >= MINIMUM_DOCKER_FREE_BYTES, "Docker storage has insufficient free space") _require(value["unit_load_state"] == "not-found", "staging service unit name already exists") _require(collision_count == 0, "staging container name already exists") @@ -491,6 +529,14 @@ def _host_readback(runner: Runner, private_ip: str) -> dict[str, Any]: "service_account": package.SERVICE_ACCOUNT, }, "docker": { + "service": { + "load_state": "loaded", + "active_state": "active", + "sub_state": "running", + "main_pid": docker_main_pid, + "n_restarts": docker_n_restarts, + "stable_across_client_observation": True, + }, "server_version": value["docker_server_version"], "server_os": "linux", "server_architecture": "amd64", @@ -535,6 +581,8 @@ def build_preflight(*, runner: Runner = _default_runner, source_revision: str) - "database_credentials_used": False, "database_queries_run": False, "control_plane_observations_before_ssh_are_describe_only": True, + "docker_service_proven_active_before_client_calls": True, + "docker_service_identity_stable_after_client_calls": True, "remote_host_runtime_mutations": False, "ssh_access_may_register_time_bounded_key": True, "ssh_key_maximum_lifetime": SSH_KEY_LIFETIME, @@ -642,6 +690,37 @@ def _write_all(descriptor: int, payload: bytes) -> None: offset += written +def _verify_receipt_entry( + parent_descriptor: int, + name: str, + expected_identity: tuple[int, int], + *, + expected_size: int, + expected_links: int, + label: str, +) -> os.stat_result: + try: + observed = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except OSError as exc: + raise PreflightError( + "receipt_publication_failed", + f"{label} cannot be identified safely", + ) from exc + if not ( + stat.S_ISREG(observed.st_mode) + and (observed.st_dev, observed.st_ino) == expected_identity + and observed.st_uid == os.geteuid() + and stat.S_IMODE(observed.st_mode) == 0o600 + and observed.st_nlink == expected_links + and observed.st_size == expected_size + ): + raise PreflightError( + "receipt_publication_failed", + f"{label} identity or posture drifted", + ) + return observed + + def _publish_private_receipt( output_path: Path, parent_descriptor: int, @@ -656,6 +735,7 @@ def _publish_private_receipt( ) from exc descriptor: int | None = None + staged_name = f".{output_path.name}.{uuid.uuid4().hex}.tmp" try: _assert_private_output_parent(parent_descriptor, output_path.parent) if _output_entry_stat(parent_descriptor, output_path.name) is not None: @@ -666,19 +746,16 @@ def _publish_private_receipt( flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW - descriptor = os.open(output_path.name, flags, 0o600, dir_fd=parent_descriptor) + descriptor = os.open(staged_name, flags, 0o600, dir_fd=parent_descriptor) os.fchmod(descriptor, 0o600) before = os.fstat(descriptor) + identity = (before.st_dev, before.st_ino) _write_all(descriptor, encoded) os.fsync(descriptor) after = os.fstat(descriptor) - named = os.stat(output_path.name, dir_fd=parent_descriptor, follow_symlinks=False) - through_path = os.stat(output_path, follow_symlinks=False) if not ( stat.S_ISREG(after.st_mode) and (before.st_dev, before.st_ino) == (after.st_dev, after.st_ino) - and (after.st_dev, after.st_ino) == (named.st_dev, named.st_ino) - and (after.st_dev, after.st_ino) == (through_path.st_dev, through_path.st_ino) and after.st_uid == os.geteuid() and stat.S_IMODE(after.st_mode) == 0o600 and after.st_nlink == 1 @@ -686,7 +763,44 @@ def _publish_private_receipt( ): raise PreflightError( "receipt_publication_failed", - "published preflight receipt identity or posture drifted", + "staged preflight receipt identity or posture drifted", + ) + _verify_receipt_entry( + parent_descriptor, + staged_name, + identity, + expected_size=len(encoded), + expected_links=1, + label="staged preflight receipt", + ) + os.link( + staged_name, + output_path.name, + src_dir_fd=parent_descriptor, + dst_dir_fd=parent_descriptor, + follow_symlinks=False, + ) + _verify_receipt_entry( + parent_descriptor, + staged_name, + identity, + expected_size=len(encoded), + expected_links=2, + label="retained staged preflight receipt", + ) + _verify_receipt_entry( + parent_descriptor, + output_path.name, + identity, + expected_size=len(encoded), + expected_links=2, + label="published preflight receipt", + ) + through_path = os.stat(output_path, follow_symlinks=False) + if (through_path.st_dev, through_path.st_ino) != identity: + raise PreflightError( + "receipt_publication_failed", + "published preflight receipt path identity drifted", ) _assert_private_output_parent(parent_descriptor, output_path.parent) os.fsync(parent_descriptor) diff --git a/tests/test_gcp_leoclean_nosend_preflight.py b/tests/test_gcp_leoclean_nosend_preflight.py index e9fdad6..75e7278 100644 --- a/tests/test_gcp_leoclean_nosend_preflight.py +++ b/tests/test_gcp_leoclean_nosend_preflight.py @@ -102,6 +102,11 @@ def live_fixtures() -> dict[str, object]: "project": "teleo-501523", "instance": "teleo-staging-1", "service_account": "sa-teleo-staging-vm@teleo-501523.iam.gserviceaccount.com", + "docker_service_load_state": "loaded", + "docker_service_active_state": "active", + "docker_service_sub_state": "running", + "docker_service_main_pid": "1701", + "docker_service_n_restarts": "0", "docker_server_version": "29.4.0", "docker_server_os": "linux", "docker_server_arch": "amd64", @@ -164,11 +169,21 @@ def test_exact_read_only_preflight_passes_without_exposing_private_ip() -> None: "database_credentials_used": False, "database_queries_run": False, "control_plane_observations_before_ssh_are_describe_only": True, + "docker_service_proven_active_before_client_calls": True, + "docker_service_identity_stable_after_client_calls": True, "remote_host_runtime_mutations": False, "ssh_access_may_register_time_bounded_key": True, "ssh_key_maximum_lifetime": "5m", } assert payload["host"]["cloudsql_private_tcp_5432"] == "reachable" + assert payload["host"]["docker"]["service"] == { + "load_state": "loaded", + "active_state": "active", + "sub_state": "running", + "main_pid": 1701, + "n_restarts": 0, + "stable_across_client_observation": True, + } assert ( payload["control_plane"]["cloudsql"]["private_ip_sha256"] == hashlib.sha256(PRIVATE_IP.encode("ascii")).hexdigest() @@ -248,7 +263,30 @@ def test_exact_read_only_preflight_passes_without_exposing_private_ip() -> None: assert "--tunnel-through-iap" in ssh assert "--ssh-key-expire-after=5m" in ssh remote = next(item.removeprefix("--command=") for item in ssh if item.startswith("--command=")) + syntax = subprocess.run(["/bin/bash", "-n"], input=remote, text=True, capture_output=True, check=False) + assert syntax.returncode == 0, syntax.stderr assert remote.startswith("set -euo pipefail\n") + first_docker_client = remote.index("sudo -n /usr/bin/docker version") + assert remote.index("docker_service_active_state_before=") < first_docker_client + assert remote.index('test "$docker_service_active_state_before" = active') < first_docker_client + assert remote.index('test "$docker_service_sub_state_before" = running') < first_docker_client + assert remote.index('test "$docker_service_main_pid_before" -gt 1') < first_docker_client + last_docker_client = remote.index("sudo -n /usr/bin/docker ps") + assert remote.index("docker_service_active_state_after=", last_docker_client) > last_docker_client + assert ( + remote.index( + 'test "$docker_service_main_pid_after" = "$docker_service_main_pid_before"', + last_docker_client, + ) + > last_docker_client + ) + assert ( + remote.index( + 'test "$docker_service_n_restarts_after" = "$docker_service_n_restarts_before"', + last_docker_client, + ) + > last_docker_client + ) for forbidden in ( "docker run", "docker pull", @@ -279,6 +317,11 @@ def test_exact_read_only_preflight_passes_without_exposing_private_ip() -> None: (("artifact", "dockerConfig"), "immutableTags", False, "not immutable"), (("cloudsql", "settings", "ipConfiguration"), "ipv4Enabled", True, "public IPv4"), (("cloudsql",), "databaseVersion", "POSTGRES_160", "exactly PostgreSQL 16"), + (("host",), "docker_service_load_state", "not-found", "not loaded"), + (("host",), "docker_service_active_state", "inactive", "not active"), + (("host",), "docker_service_sub_state", "dead", "not running"), + (("host",), "docker_service_main_pid", "0", "MainPID"), + (("host",), "docker_service_n_restarts", "-1", "restart count"), (("host",), "container_collision_count", "1", "container name"), (("host",), "unit_load_state", "loaded", "unit name"), (("host",), "docker_root_free_bytes", "1024", "free space"), @@ -499,7 +542,10 @@ def test_cli_publishes_one_private_no_clobber_receipt( assert stat.S_ISREG(observed.st_mode) assert stat.S_IMODE(observed.st_mode) == 0o600 assert observed.st_uid == os.geteuid() - assert observed.st_nlink == 1 + assert observed.st_nlink == 2 + retained = list(output.parent.glob(".preflight.json.*.tmp")) + assert len(retained) == 1 + assert retained[0].stat().st_ino == observed.st_ino @pytest.mark.parametrize("entry_kind", ["file", "directory", "symlink"]) @@ -578,6 +624,7 @@ def test_publication_race_fails_without_overwriting_the_created_entry( assert preflight.main(["--output", str(output)]) == 1 assert output.read_text(encoding="utf-8") == "replacement-sentinel\n" + assert list(output.parent.glob(".preflight.json.*.tmp")) == [] assert json.loads(capsys.readouterr().out) == { "category": "receipt_publication_failed", "schema": preflight.SCHEMA, @@ -585,7 +632,35 @@ def test_publication_race_fails_without_overwriting_the_created_entry( } -def test_publication_failure_is_bounded_and_preserves_created_residue( +def test_link_race_preserves_the_complete_staged_inode_without_overwrite( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + output = _private_output_directory(tmp_path) / "preflight.json" + parent_descriptor = preflight._open_private_output_parent(output) + payload = {"schema": preflight.SCHEMA, "status": "pass", "source_revision": REVISION} + real_link = preflight.os.link + + def race_link(source: str, destination: str, **kwargs: object) -> None: + output.write_text("replacement-sentinel\n", encoding="utf-8") + real_link(source, destination, **kwargs) + + monkeypatch.setattr(preflight.os, "link", race_link) + try: + with pytest.raises(preflight.PreflightError) as caught: + preflight._publish_private_receipt(output, parent_descriptor, payload) + finally: + os.close(parent_descriptor) + + assert caught.value.category == "receipt_publication_failed" + assert output.read_text(encoding="utf-8") == "replacement-sentinel\n" + retained = list(output.parent.glob(".preflight.json.*.tmp")) + assert len(retained) == 1 + assert json.loads(retained[0].read_text(encoding="utf-8")) == payload + assert retained[0].stat().st_nlink == 1 + + +def test_prepublication_fsync_failure_is_bounded_and_preserves_only_staged_residue( tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, @@ -611,6 +686,42 @@ def test_publication_failure_is_bounded_and_preserves_created_residue( "status": "blocked", } assert sentinel not in stdout - assert output.is_file() - assert stat.S_IMODE(output.stat().st_mode) == 0o600 - assert json.loads(output.read_text(encoding="utf-8"))["status"] == "pass" + assert not output.exists() + retained = list(output.parent.glob(".preflight.json.*.tmp")) + assert len(retained) == 1 + assert stat.S_IMODE(retained[0].stat().st_mode) == 0o600 + assert retained[0].stat().st_nlink == 1 + assert json.loads(retained[0].read_text(encoding="utf-8"))["status"] == "pass" + + +def test_prepublication_write_failure_is_bounded_and_never_creates_canonical_output( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + _stub_successful_cli(monkeypatch) + output = _private_output_directory(tmp_path) / "preflight.json" + sentinel = "private-write-fault-sentinel" + + def fail_after_partial_write(descriptor: int, payload: bytes) -> None: + os.write(descriptor, payload[:16]) + raise OSError(sentinel) + + monkeypatch.setattr(preflight, "_write_all", fail_after_partial_write) + + assert preflight.main(["--output", str(output)]) == 1 + + stdout = capsys.readouterr().out + assert json.loads(stdout) == { + "category": "receipt_publication_failed", + "schema": preflight.SCHEMA, + "status": "blocked", + } + assert sentinel not in stdout + assert not output.exists() + retained = list(output.parent.glob(".preflight.json.*.tmp")) + assert len(retained) == 1 + assert stat.S_IMODE(retained[0].stat().st_mode) == 0o600 + assert retained[0].stat().st_nlink == 1 + with pytest.raises(json.JSONDecodeError): + json.loads(retained[0].read_text(encoding="utf-8"))