Close staging preflight review gaps

docs/gcp-leoclean-nosend-preflight.md

@@ -12,7 +12,11 @@ architecture, private-only VM attachment to `teleo-staging-europe-west6`
 (`10.60.0.0/20`), Private Google Access, immutable Docker Artifact
 Registry repository, exact private PostgreSQL 16 Cloud SQL endpoint, host
 architecture, Docker server, Docker storage floor, unused service/container
-names, and private TCP/5432 reachability from the VM.
+names, and private TCP/5432 reachability from the VM. Before the first Docker
+client command, it requires `docker.service` to be loaded, active, and running;
+after the final client command it requires unchanged service state, `MainPID`,
+and restart count. This prevents a read probe from activating Docker through
+`docker.socket` or silently accepting a daemon restart.
 
 ## Authority boundary
 
@@ -42,10 +46,13 @@ python3 ops/check_gcp_leoclean_nosend_preflight.py \
 The output parent must already exist, be owned by the operator, and have no
 group/other write permission. The output name must not exist. The checker
 creates it once with no-clobber/no-symlink semantics, keeps the same file and
-directory identity open through publication, writes mode `0600`, and fsyncs
+directory identity open through publication, writes and fsyncs a mode-`0600`
-the file and directory. It never removes or overwrites the output pathname;
+private staged inode, and only then hard-links that complete inode to the
-an ambiguous publication failure can therefore leave a private residue for
+canonical output name without replacement. The retained staged and canonical
-operator inspection.
+names are two links to the same inode; they are never removed automatically.
+A pre-publication failure therefore leaves only the private staged residue,
+not a valid-looking canonical pass receipt. A later ambiguous publication
+failure preserves the exact retained inode for operator inspection.
 
 Authentication, permission, connection, and receipt-publication failures emit
 only a bounded category; command stderr and filesystem exception details are

docs/workstreams/gcp-leoclean/goals.md

@@ -73,19 +73,22 @@ cut over safely and retire the VPS after soak.
   bounded IAP/SSH path uses a fixed remote command and a maximum five-minute
   key lifetime. Receipt publication now requires a pre-existing private
   operator-owned directory, creates one mode-`0600` no-clobber/no-symlink
-  file through a held directory descriptor, verifies stable identity, fsyncs
+  staged inode through a held directory descriptor, fsyncs it before a
-  both file and directory, and preserves ambiguous residues. No live run is
+  no-clobber canonical hard link, and retains both links without automatic
-  currently authorized.
+  cleanup. The host probe proves Docker loaded/active/running before its first
+  client call and unchanged service state, `MainPID`, and restart count after
+  the last, preventing socket activation from changing runtime state. No live
+  run is currently authorized.
 - Artifact Registry cleanup policy is not an additional hard gate for the
   package: immutable tags prevent deletion of the retained tagged candidate.
   The post-push/finalization path must still re-prove the exact retained
   candidate tag-to-digest binding immediately before deployment.
-- Current offline evidence for the preflight extraction: 243 focused
+- Current offline evidence for the preflight extraction: 250 focused
   preflight/package/OCI tests pass; Ruff, Python compilation, formatting, and
-  diff checks pass. The repository suite produced 2,497 passes and 5 expected
+  diff checks pass. The repository suite produced 2,504 passes and 5 expected
   skips inside the desktop sandbox; all 18 sandbox-denied Docker, loopback, and
   Swift-cache cases passed in a separately authorized 59-test local rerun,
-  reconciling the suite to 2,515 passes and 5 skips. No cloud or registry
+  reconciling the suite to 2,522 passes and 5 skips. No cloud or registry
   endpoint was contacted.
 - Historical only: the superseded preflight branch records an earlier live
   observation of `e2-standard-4` on `teleo-staging-europe-west6`

ops/check_gcp_leoclean_nosend_preflight.py

@@ -14,6 +14,7 @@ import shlex
 import stat
 import subprocess
 import sys
+import uuid
 from collections.abc import Callable
 from datetime import UTC, datetime
 from pathlib import Path
@@ -397,6 +398,16 @@ project=\"$(metadata project/project-id)\"
 instance=\"$(metadata instance/name)\"
 service_account=\"$(metadata instance/service-accounts/default/email)\"
 test -x /usr/bin/docker
+docker_service_load_state_before=\"$(systemctl show docker.service --property=LoadState --value)\"
+docker_service_active_state_before=\"$(systemctl show docker.service --property=ActiveState --value)\"
+docker_service_sub_state_before=\"$(systemctl show docker.service --property=SubState --value)\"
+docker_service_main_pid_before=\"$(systemctl show docker.service --property=MainPID --value)\"
+docker_service_n_restarts_before=\"$(systemctl show docker.service --property=NRestarts --value)\"
+test \"$docker_service_load_state_before\" = loaded
+test \"$docker_service_active_state_before\" = active
+test \"$docker_service_sub_state_before\" = running
+test \"$docker_service_main_pid_before\" -gt 1
+test \"$docker_service_n_restarts_before\" -ge 0
 docker_server_version=\"$(sudo -n /usr/bin/docker version --format '{{{{.Server.Version}}}}')\"
 docker_server_os=\"$(sudo -n /usr/bin/docker version --format '{{{{.Server.Os}}}}')\"
 docker_server_arch=\"$(sudo -n /usr/bin/docker version --format '{{{{.Server.Arch}}}}')\"
@@ -404,11 +415,26 @@ docker_root=\"$(sudo -n /usr/bin/docker info --format '{{{{.DockerRootDir}}}}')\
 docker_root_free_bytes=\"$(sudo -n /usr/bin/df -B1 --output=avail \"$docker_root\" | tail -n 1 | tr -d ' ')\"
 unit_load_state=\"$(systemctl show {shlex.quote(package.SERVICE)} --property=LoadState --value)\"
 container_collision_count=\"$(sudo -n /usr/bin/docker ps -aq --filter 'name=^/{package.CONTAINER_NAME}$' | wc -l | tr -d ' ')\"
+docker_service_load_state_after=\"$(systemctl show docker.service --property=LoadState --value)\"
+docker_service_active_state_after=\"$(systemctl show docker.service --property=ActiveState --value)\"
+docker_service_sub_state_after=\"$(systemctl show docker.service --property=SubState --value)\"
+docker_service_main_pid_after=\"$(systemctl show docker.service --property=MainPID --value)\"
+docker_service_n_restarts_after=\"$(systemctl show docker.service --property=NRestarts --value)\"
+test \"$docker_service_load_state_after\" = \"$docker_service_load_state_before\"
+test \"$docker_service_active_state_after\" = \"$docker_service_active_state_before\"
+test \"$docker_service_sub_state_after\" = \"$docker_service_sub_state_before\"
+test \"$docker_service_main_pid_after\" = \"$docker_service_main_pid_before\"
+test \"$docker_service_n_restarts_after\" = \"$docker_service_n_restarts_before\"
 if timeout 5 bash -c 'exec 3<>/dev/tcp/{private_ip}/5432; exec 3<&-; exec 3>&-'; then cloudsql_tcp=reachable; else cloudsql_tcp=unreachable; fi
 printf 'architecture=%s\n' \"$architecture\"
 printf 'project=%s\n' \"$project\"
 printf 'instance=%s\n' \"$instance\"
 printf 'service_account=%s\n' \"$service_account\"
+printf 'docker_service_load_state=%s\n' \"$docker_service_load_state_after\"
+printf 'docker_service_active_state=%s\n' \"$docker_service_active_state_after\"
+printf 'docker_service_sub_state=%s\n' \"$docker_service_sub_state_after\"
+printf 'docker_service_main_pid=%s\n' \"$docker_service_main_pid_after\"
+printf 'docker_service_n_restarts=%s\n' \"$docker_service_n_restarts_after\"
 printf 'docker_server_version=%s\n' \"$docker_server_version\"
 printf 'docker_server_os=%s\n' \"$docker_server_os\"
 printf 'docker_server_arch=%s\n' \"$docker_server_arch\"
@@ -448,6 +474,11 @@ def _parse_host_output(output: str) -> dict[str, str]:
         "project",
         "instance",
         "service_account",
+        "docker_service_load_state",
+        "docker_service_active_state",
+        "docker_service_sub_state",
+        "docker_service_main_pid",
+        "docker_service_n_restarts",
         "docker_server_version",
         "docker_server_os",
         "docker_server_arch",
@@ -477,8 +508,15 @@ def _host_readback(runner: Runner, private_ip: str) -> dict[str, Any]:
     try:
         free_bytes = int(value["docker_root_free_bytes"])
         collision_count = int(value["container_collision_count"])
+        docker_main_pid = int(value["docker_service_main_pid"])
+        docker_n_restarts = int(value["docker_service_n_restarts"])
     except ValueError as exc:
         raise PreflightError("invalid_readback", "host numeric field was invalid") from exc
+    _require(value["docker_service_load_state"] == "loaded", "Docker service is not loaded")
+    _require(value["docker_service_active_state"] == "active", "Docker service is not active")
+    _require(value["docker_service_sub_state"] == "running", "Docker service is not running")
+    _require(docker_main_pid > 1, "Docker service MainPID is invalid")
+    _require(docker_n_restarts >= 0, "Docker service restart count is invalid")
     _require(free_bytes >= MINIMUM_DOCKER_FREE_BYTES, "Docker storage has insufficient free space")
     _require(value["unit_load_state"] == "not-found", "staging service unit name already exists")
     _require(collision_count == 0, "staging container name already exists")
@@ -491,6 +529,14 @@ def _host_readback(runner: Runner, private_ip: str) -> dict[str, Any]:
             "service_account": package.SERVICE_ACCOUNT,
         },
         "docker": {
+            "service": {
+                "load_state": "loaded",
+                "active_state": "active",
+                "sub_state": "running",
+                "main_pid": docker_main_pid,
+                "n_restarts": docker_n_restarts,
+                "stable_across_client_observation": True,
+            },
             "server_version": value["docker_server_version"],
             "server_os": "linux",
             "server_architecture": "amd64",
@@ -535,6 +581,8 @@ def build_preflight(*, runner: Runner = _default_runner, source_revision: str) -
             "database_credentials_used": False,
             "database_queries_run": False,
             "control_plane_observations_before_ssh_are_describe_only": True,
+            "docker_service_proven_active_before_client_calls": True,
+            "docker_service_identity_stable_after_client_calls": True,
             "remote_host_runtime_mutations": False,
             "ssh_access_may_register_time_bounded_key": True,
             "ssh_key_maximum_lifetime": SSH_KEY_LIFETIME,
@@ -642,6 +690,37 @@ def _write_all(descriptor: int, payload: bytes) -> None:
         offset += written
 
 
+def _verify_receipt_entry(
+    parent_descriptor: int,
+    name: str,
+    expected_identity: tuple[int, int],
+    *,
+    expected_size: int,
+    expected_links: int,
+    label: str,
+) -> os.stat_result:
+    try:
+        observed = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False)
+    except OSError as exc:
+        raise PreflightError(
+            "receipt_publication_failed",
+            f"{label} cannot be identified safely",
+        ) from exc
+    if not (
+        stat.S_ISREG(observed.st_mode)
+        and (observed.st_dev, observed.st_ino) == expected_identity
+        and observed.st_uid == os.geteuid()
+        and stat.S_IMODE(observed.st_mode) == 0o600
+        and observed.st_nlink == expected_links
+        and observed.st_size == expected_size
+    ):
+        raise PreflightError(
+            "receipt_publication_failed",
+            f"{label} identity or posture drifted",
+        )
+    return observed
+
+
 def _publish_private_receipt(
     output_path: Path,
     parent_descriptor: int,
@@ -656,6 +735,7 @@ def _publish_private_receipt(
         ) from exc
 
     descriptor: int | None = None
+    staged_name = f".{output_path.name}.{uuid.uuid4().hex}.tmp"
     try:
         _assert_private_output_parent(parent_descriptor, output_path.parent)
         if _output_entry_stat(parent_descriptor, output_path.name) is not None:
@@ -666,19 +746,16 @@ def _publish_private_receipt(
         flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
         if hasattr(os, "O_NOFOLLOW"):
             flags |= os.O_NOFOLLOW
-        descriptor = os.open(output_path.name, flags, 0o600, dir_fd=parent_descriptor)
+        descriptor = os.open(staged_name, flags, 0o600, dir_fd=parent_descriptor)
         os.fchmod(descriptor, 0o600)
         before = os.fstat(descriptor)
+        identity = (before.st_dev, before.st_ino)
         _write_all(descriptor, encoded)
         os.fsync(descriptor)
         after = os.fstat(descriptor)
-        named = os.stat(output_path.name, dir_fd=parent_descriptor, follow_symlinks=False)
-        through_path = os.stat(output_path, follow_symlinks=False)
         if not (
             stat.S_ISREG(after.st_mode)
             and (before.st_dev, before.st_ino) == (after.st_dev, after.st_ino)
-            and (after.st_dev, after.st_ino) == (named.st_dev, named.st_ino)
-            and (after.st_dev, after.st_ino) == (through_path.st_dev, through_path.st_ino)
             and after.st_uid == os.geteuid()
             and stat.S_IMODE(after.st_mode) == 0o600
             and after.st_nlink == 1
@@ -686,7 +763,44 @@ def _publish_private_receipt(
         ):
             raise PreflightError(
                 "receipt_publication_failed",
-                "published preflight receipt identity or posture drifted",
+                "staged preflight receipt identity or posture drifted",
+            )
+        _verify_receipt_entry(
+            parent_descriptor,
+            staged_name,
+            identity,
+            expected_size=len(encoded),
+            expected_links=1,
+            label="staged preflight receipt",
+        )
+        os.link(
+            staged_name,
+            output_path.name,
+            src_dir_fd=parent_descriptor,
+            dst_dir_fd=parent_descriptor,
+            follow_symlinks=False,
+        )
+        _verify_receipt_entry(
+            parent_descriptor,
+            staged_name,
+            identity,
+            expected_size=len(encoded),
+            expected_links=2,
+            label="retained staged preflight receipt",
+        )
+        _verify_receipt_entry(
+            parent_descriptor,
+            output_path.name,
+            identity,
+            expected_size=len(encoded),
+            expected_links=2,
+            label="published preflight receipt",
+        )
+        through_path = os.stat(output_path, follow_symlinks=False)
+        if (through_path.st_dev, through_path.st_ino) != identity:
+            raise PreflightError(
+                "receipt_publication_failed",
+                "published preflight receipt path identity drifted",
             )
         _assert_private_output_parent(parent_descriptor, output_path.parent)
         os.fsync(parent_descriptor)

tests/test_gcp_leoclean_nosend_preflight.py

@@ -102,6 +102,11 @@ def live_fixtures() -> dict[str, object]:
             "project": "teleo-501523",
             "instance": "teleo-staging-1",
             "service_account": "sa-teleo-staging-vm@teleo-501523.iam.gserviceaccount.com",
+            "docker_service_load_state": "loaded",
+            "docker_service_active_state": "active",
+            "docker_service_sub_state": "running",
+            "docker_service_main_pid": "1701",
+            "docker_service_n_restarts": "0",
             "docker_server_version": "29.4.0",
             "docker_server_os": "linux",
             "docker_server_arch": "amd64",
@@ -164,11 +169,21 @@ def test_exact_read_only_preflight_passes_without_exposing_private_ip() -> None:
         "database_credentials_used": False,
         "database_queries_run": False,
         "control_plane_observations_before_ssh_are_describe_only": True,
+        "docker_service_proven_active_before_client_calls": True,
+        "docker_service_identity_stable_after_client_calls": True,
         "remote_host_runtime_mutations": False,
         "ssh_access_may_register_time_bounded_key": True,
         "ssh_key_maximum_lifetime": "5m",
     }
     assert payload["host"]["cloudsql_private_tcp_5432"] == "reachable"
+    assert payload["host"]["docker"]["service"] == {
+        "load_state": "loaded",
+        "active_state": "active",
+        "sub_state": "running",
+        "main_pid": 1701,
+        "n_restarts": 0,
+        "stable_across_client_observation": True,
+    }
     assert (
         payload["control_plane"]["cloudsql"]["private_ip_sha256"]
         == hashlib.sha256(PRIVATE_IP.encode("ascii")).hexdigest()
@@ -248,7 +263,30 @@ def test_exact_read_only_preflight_passes_without_exposing_private_ip() -> None:
     assert "--tunnel-through-iap" in ssh
     assert "--ssh-key-expire-after=5m" in ssh
     remote = next(item.removeprefix("--command=") for item in ssh if item.startswith("--command="))
+    syntax = subprocess.run(["/bin/bash", "-n"], input=remote, text=True, capture_output=True, check=False)
+    assert syntax.returncode == 0, syntax.stderr
     assert remote.startswith("set -euo pipefail\n")
+    first_docker_client = remote.index("sudo -n /usr/bin/docker version")
+    assert remote.index("docker_service_active_state_before=") < first_docker_client
+    assert remote.index('test "$docker_service_active_state_before" = active') < first_docker_client
+    assert remote.index('test "$docker_service_sub_state_before" = running') < first_docker_client
+    assert remote.index('test "$docker_service_main_pid_before" -gt 1') < first_docker_client
+    last_docker_client = remote.index("sudo -n /usr/bin/docker ps")
+    assert remote.index("docker_service_active_state_after=", last_docker_client) > last_docker_client
+    assert (
+        remote.index(
+            'test "$docker_service_main_pid_after" = "$docker_service_main_pid_before"',
+            last_docker_client,
+        )
+        > last_docker_client
+    )
+    assert (
+        remote.index(
+            'test "$docker_service_n_restarts_after" = "$docker_service_n_restarts_before"',
+            last_docker_client,
+        )
+        > last_docker_client
+    )
     for forbidden in (
         "docker run",
         "docker pull",
@@ -279,6 +317,11 @@ def test_exact_read_only_preflight_passes_without_exposing_private_ip() -> None:
         (("artifact", "dockerConfig"), "immutableTags", False, "not immutable"),
         (("cloudsql", "settings", "ipConfiguration"), "ipv4Enabled", True, "public IPv4"),
         (("cloudsql",), "databaseVersion", "POSTGRES_160", "exactly PostgreSQL 16"),
+        (("host",), "docker_service_load_state", "not-found", "not loaded"),
+        (("host",), "docker_service_active_state", "inactive", "not active"),
+        (("host",), "docker_service_sub_state", "dead", "not running"),
+        (("host",), "docker_service_main_pid", "0", "MainPID"),
+        (("host",), "docker_service_n_restarts", "-1", "restart count"),
         (("host",), "container_collision_count", "1", "container name"),
         (("host",), "unit_load_state", "loaded", "unit name"),
         (("host",), "docker_root_free_bytes", "1024", "free space"),
@@ -499,7 +542,10 @@ def test_cli_publishes_one_private_no_clobber_receipt(
     assert stat.S_ISREG(observed.st_mode)
     assert stat.S_IMODE(observed.st_mode) == 0o600
     assert observed.st_uid == os.geteuid()
-    assert observed.st_nlink == 1
+    assert observed.st_nlink == 2
+    retained = list(output.parent.glob(".preflight.json.*.tmp"))
+    assert len(retained) == 1
+    assert retained[0].stat().st_ino == observed.st_ino
 
 
 @pytest.mark.parametrize("entry_kind", ["file", "directory", "symlink"])
@@ -578,6 +624,7 @@ def test_publication_race_fails_without_overwriting_the_created_entry(
     assert preflight.main(["--output", str(output)]) == 1
 
     assert output.read_text(encoding="utf-8") == "replacement-sentinel\n"
+    assert list(output.parent.glob(".preflight.json.*.tmp")) == []
     assert json.loads(capsys.readouterr().out) == {
         "category": "receipt_publication_failed",
         "schema": preflight.SCHEMA,
@@ -585,7 +632,35 @@ def test_publication_race_fails_without_overwriting_the_created_entry(
     }
 
 
-def test_publication_failure_is_bounded_and_preserves_created_residue(
+def test_link_race_preserves_the_complete_staged_inode_without_overwrite(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    output = _private_output_directory(tmp_path) / "preflight.json"
+    parent_descriptor = preflight._open_private_output_parent(output)
+    payload = {"schema": preflight.SCHEMA, "status": "pass", "source_revision": REVISION}
+    real_link = preflight.os.link
+
+    def race_link(source: str, destination: str, **kwargs: object) -> None:
+        output.write_text("replacement-sentinel\n", encoding="utf-8")
+        real_link(source, destination, **kwargs)
+
+    monkeypatch.setattr(preflight.os, "link", race_link)
+    try:
+        with pytest.raises(preflight.PreflightError) as caught:
+            preflight._publish_private_receipt(output, parent_descriptor, payload)
+    finally:
+        os.close(parent_descriptor)
+
+    assert caught.value.category == "receipt_publication_failed"
+    assert output.read_text(encoding="utf-8") == "replacement-sentinel\n"
+    retained = list(output.parent.glob(".preflight.json.*.tmp"))
+    assert len(retained) == 1
+    assert json.loads(retained[0].read_text(encoding="utf-8")) == payload
+    assert retained[0].stat().st_nlink == 1
+
+
+def test_prepublication_fsync_failure_is_bounded_and_preserves_only_staged_residue(
     tmp_path: Path,
     capsys: pytest.CaptureFixture[str],
     monkeypatch: pytest.MonkeyPatch,
@@ -611,6 +686,42 @@ def test_publication_failure_is_bounded_and_preserves_created_residue(
         "status": "blocked",
     }
     assert sentinel not in stdout
-    assert output.is_file()
+    assert not output.exists()
-    assert stat.S_IMODE(output.stat().st_mode) == 0o600
+    retained = list(output.parent.glob(".preflight.json.*.tmp"))
-    assert json.loads(output.read_text(encoding="utf-8"))["status"] == "pass"
+    assert len(retained) == 1
+    assert stat.S_IMODE(retained[0].stat().st_mode) == 0o600
+    assert retained[0].stat().st_nlink == 1
+    assert json.loads(retained[0].read_text(encoding="utf-8"))["status"] == "pass"
+
+
+def test_prepublication_write_failure_is_bounded_and_never_creates_canonical_output(
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    _stub_successful_cli(monkeypatch)
+    output = _private_output_directory(tmp_path) / "preflight.json"
+    sentinel = "private-write-fault-sentinel"
+
+    def fail_after_partial_write(descriptor: int, payload: bytes) -> None:
+        os.write(descriptor, payload[:16])
+        raise OSError(sentinel)
+
+    monkeypatch.setattr(preflight, "_write_all", fail_after_partial_write)
+
+    assert preflight.main(["--output", str(output)]) == 1
+
+    stdout = capsys.readouterr().out
+    assert json.loads(stdout) == {
+        "category": "receipt_publication_failed",
+        "schema": preflight.SCHEMA,
+        "status": "blocked",
+    }
+    assert sentinel not in stdout
+    assert not output.exists()
+    retained = list(output.parent.glob(".preflight.json.*.tmp"))
+    assert len(retained) == 1
+    assert stat.S_IMODE(retained[0].stat().st_mode) == 0o600
+    assert retained[0].stat().st_nlink == 1
+    with pytest.raises(json.JSONDecodeError):
+        json.loads(retained[0].read_text(encoding="utf-8"))