From f0719613b6c4127074432bdfd5526f0a4e0e781d Mon Sep 17 00:00:00 2001 From: fwazb Date: Thu, 23 Jul 2026 10:41:40 -0700 Subject: [PATCH] Harden leoclean first-start readiness --- ops/install_gcp_leoclean_nosend_release.py | 135 ++++++- ops/leoclean_nosend_service_control.py | 75 +++- tests/test_gcp_leoclean_nosend_installer.py | 391 +++++++++++++++++++- 3 files changed, 575 insertions(+), 26 deletions(-) diff --git a/ops/install_gcp_leoclean_nosend_release.py b/ops/install_gcp_leoclean_nosend_release.py index 785d793..8395f54 100644 --- a/ops/install_gcp_leoclean_nosend_release.py +++ b/ops/install_gcp_leoclean_nosend_release.py @@ -28,6 +28,7 @@ if str(CONTRACT_ROOT) not in sys.path: sys.path.insert(0, str(CONTRACT_ROOT)) from ops import gcp_leoclean_nosend_package as package # noqa: E402 +from ops import leoclean_nosend_service_control as service_control # noqa: E402 INSTALL_RECEIPT_SCHEMA = "livingip.leocleanNoSendInstallReceipt.v2" ROLLBACK_STATE_SCHEMA = "livingip.leocleanNoSendRollbackState.v1" @@ -142,6 +143,7 @@ class InstallPaths: receipt: Path rollback_state: Path lock: Path + cidfile: Path proc_root: Path dropin_directories: tuple[Path, ...] @@ -163,6 +165,7 @@ class InstallPaths: receipt=state / "install-receipt.json", rollback_state=state / "rollback-state.json", lock=rooted("/run/leoclean-gcp-nosend-installer.lock"), + cidfile=rooted(package.SERVICE_CIDFILE), proc_root=proc_root or rooted("/proc"), dropin_directories=tuple( rooted(path) @@ -940,6 +943,36 @@ def _docker_inspect(arguments: list[str], runner: Runner, paths: InstallPaths, l return value[0] +def _require_candidate_absent( + *, + runner: Runner, + paths: InstallPaths, + sleeper: Sleeper, +) -> None: + for readback in range(2): + _require( + not paths.cidfile.exists() and not paths.cidfile.is_symlink(), + "candidate container ID file remained after stop", + ) + reference = package.CONTAINER_NAME + completed = _run( + [*_docker_prefix(paths), "container", "inspect", reference], + runner, + label="stopped candidate inspection", + ) + _require(completed.returncode != 0, "candidate container remained after stop") + try: + error = completed.stderr.decode("utf-8", "strict").strip() + except UnicodeError as exc: + raise InstallError("candidate container absence could not be proven") from exc + absence = re.compile( + rf"^(?:Error response from daemon: |Error: )?No such (?:container|object): {re.escape(reference)}$" + ) + _require(absence.fullmatch(error) is not None, "candidate container absence could not be proven") + if readback == 0: + sleeper(0.05) + + def _validate_local_image(release: dict[str, Any], runner: Runner, paths: InstallPaths) -> None: reference = release["image"]["reference"] inspected = _docker_inspect(["image", "inspect", reference], runner, paths, "local image inspection") @@ -951,7 +984,12 @@ def _validate_local_image(release: dict[str, Any], runner: Runner, paths: Instal _require(normalized == release["inspection"], "local image differed from the finalized inspection") -def _validate_running_container(value: dict[str, Any], release: dict[str, Any]) -> dict[str, Any]: +def _validate_starting_container( + value: dict[str, Any], + release: dict[str, Any], + *, + expected_container_id: str, +) -> dict[str, Any] | None: config = value.get("Config") state = value.get("State") health = state.get("Health") if isinstance(state, dict) else None @@ -959,9 +997,10 @@ def _validate_running_container(value: dict[str, Any], release: dict[str, Any]) labels = config.get("Labels") if isinstance(config, dict) else None runtime = release["inspection"]["runtime_config"] container_id = value.get("Id") - _require(isinstance(container_id, str) and HEX_64.fullmatch(container_id) is not None, "container ID was invalid") _require( - value.get("Name") == f"/{package.CONTAINER_NAME}" + container_id == expected_container_id + and HEX_64.fullmatch(expected_container_id) is not None + and value.get("Name") == f"/{package.CONTAINER_NAME}" and value.get("Image") == release["image"]["config_digest"] and isinstance(config, dict) and config.get("Image") == release["image"]["reference"] @@ -970,11 +1009,6 @@ def _validate_running_container(value: dict[str, Any], release: dict[str, Any]) and labels.get("livingip.release.sha256") == release["release_sha256"] and labels.get("livingip.container.role") == "leoclean-gcp-nosend" and isinstance(state, dict) - and state.get("Running") is True - and isinstance(state.get("Pid"), int) - and state["Pid"] > 1 - and isinstance(health, dict) - and health.get("Status") == "healthy" and isinstance(host, dict) and host.get("ReadonlyRootfs") is True and host.get("NetworkMode") == "bridge" @@ -990,16 +1024,39 @@ def _validate_running_container(value: dict[str, Any], release: dict[str, Any]) value.get("Path") == expected_path and value.get("Args") == expected_args, "running container command differed from the release contract", ) + running = state.get("Running") + pid = state.get("Pid") + status = state.get("Status") + _require(isinstance(running, bool) and isinstance(pid, int), "container startup state was invalid") + if running is False: + _require(status == "created" and pid == 0, "container exited before becoming healthy") + return None + _require(pid > 1 and isinstance(health, dict), "container startup state was invalid") + health_status = health.get("Status") + _require(health_status in {"starting", "healthy"}, "container became unhealthy during startup") + if health_status == "starting": + return None return {"container_id": container_id, "container_pid": state["Pid"], "health": "healthy"} +def _read_startup_container_id(paths: InstallPaths) -> str | None: + try: + record = service_control._read_cidfile(paths.cidfile) + except service_control.ControlError as exc: + raise InstallError("container ID file was unsafe or invalid") from exc + if record is None: + return None + container_id, _identity = record + return container_id + + def verify_running_release( release: dict[str, Any], *, runner: Runner, paths: InstallPaths, sleeper: Sleeper = time.sleep, - attempts: int = 30, + attempts: int = 240, ) -> dict[str, Any]: first: dict[str, Any] | None = None for _attempt in range(attempts): @@ -1019,14 +1076,40 @@ def verify_running_release( release, str(first["ControlGroup"]), ) - first_container = _docker_inspect( - ["container", "inspect", package.CONTAINER_NAME], - runner, - paths, - "running container inspection", - ) - _require(first_container is not None, "running container was absent") - container = _validate_running_container(first_container, release) + container: dict[str, Any] | None = None + for _attempt in range(attempts): + current = _service_state(runner) + _require( + current["MainPID"] == first["MainPID"] + and current["NRestarts"] == first["NRestarts"] + and current["InvocationID"] == first["InvocationID"] + and current["ControlGroup"] == first["ControlGroup"] + and current["ActiveState"] == "active" + and current["SubState"] == "running", + "service changed during startup verification", + ) + container_id = _read_startup_container_id(paths) + if container_id is None: + sleeper(0.25) + continue + first_container = _docker_inspect( + ["container", "inspect", package.CONTAINER_NAME], + runner, + paths, + "running container inspection", + ) + if first_container is None: + sleeper(0.25) + continue + container = _validate_starting_container( + first_container, + release, + expected_container_id=container_id, + ) + if container is not None: + break + sleeper(0.25) + _require(container is not None, "container did not become healthy within the bounded wait") _validate_local_image(release, runner, paths) second_container = _docker_inspect( ["container", "inspect", container["container_id"]], @@ -1035,7 +1118,12 @@ def verify_running_release( "stable container inspection", ) _require(second_container is not None, "running container exited during verification") - stable_container = _validate_running_container(second_container, release) + stable_container = _validate_starting_container( + second_container, + release, + expected_container_id=container["container_id"], + ) + _require(stable_container is not None, "running container changed during verification") second = _service_state(runner) _validate_unit_configuration(second, paths, release) stable_supervising_process = _validate_supervising_process( @@ -1044,6 +1132,7 @@ def verify_running_release( release, str(second["ControlGroup"]), ) + stable_container_id = _read_startup_container_id(paths) _require( first["MainPID"] == second["MainPID"] and first["NRestarts"] == second["NRestarts"] @@ -1052,6 +1141,7 @@ def verify_running_release( and first["ActiveState"] == second["ActiveState"] == "active" and first["SubState"] == second["SubState"] == "running" and container == stable_container + and stable_container_id == container["container_id"] and supervising_process == stable_supervising_process, "service or container changed during verification", ) @@ -1561,8 +1651,13 @@ def _install_release_transaction( if candidate_may_be_active: try: _systemctl("stop", runner) - except InstallError: - rollback_errors.append("candidate_stop") + _require_candidate_absent( + runner=runner, + paths=paths, + sleeper=sleeper, + ) + except InstallError as stop_exc: + raise InstallError("installation failed and automatic rollback was incomplete") from stop_exc for path in (paths.receipt, paths.rollback_state, paths.release, paths.unit, paths.helper): if path not in attempted_paths: continue diff --git a/ops/leoclean_nosend_service_control.py b/ops/leoclean_nosend_service_control.py index f14b9a2..bba2a71 100644 --- a/ops/leoclean_nosend_service_control.py +++ b/ops/leoclean_nosend_service_control.py @@ -24,6 +24,7 @@ IMAGE_REPOSITORY = "europe-west6-docker.pkg.dev/teleo-501523/teleo/leoclean-nose SAFE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" MAX_DOCKER_OUTPUT_BYTES = 1024 * 1024 HEX_64 = re.compile(r"^[0-9a-f]{64}$") +HEX_PARTIAL = re.compile(r"^[0-9a-f]{0,63}$") IMAGE_REFERENCE = re.compile(rf"^{re.escape(IMAGE_REPOSITORY)}@sha256:[0-9a-f]{{64}}$") IMAGE_CONFIG_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") @@ -160,7 +161,7 @@ def _cidfile_parent_descriptor(cidfile: Path) -> int: return descriptor -def _read_cidfile(cidfile: Path) -> tuple[str, tuple[int, int]] | None: +def _read_cidfile(cidfile: Path) -> tuple[str | None, tuple[int, int]] | None: parent_descriptor = _cidfile_parent_descriptor(cidfile) descriptor: int | None = None try: @@ -187,8 +188,10 @@ def _read_cidfile(cidfile: Path) -> tuple[str, tuple[int, int]] | None: container_id = raw.decode("ascii", "strict").strip() except UnicodeError as exc: raise ControlError("container ID file was invalid") from exc - _require(HEX_64.fullmatch(container_id) is not None, "container ID file was invalid") - return container_id, (observed.st_dev, observed.st_ino) + if HEX_64.fullmatch(container_id) is not None: + return container_id, (observed.st_dev, observed.st_ino) + _require(HEX_PARTIAL.fullmatch(container_id) is not None, "container ID file was invalid") + return None, (observed.st_dev, observed.st_ino) except OSError as exc: raise ControlError("container ID file could not be read") from exc finally: @@ -231,6 +234,52 @@ def _stop_exact(container_id: str, runner: Runner, sleeper: Callable[[float], No _wait_absent(container_id, runner, sleeper) +def _remove_created_exact( + container_id: str, + *, + release_sha256: str, + image_reference: str, + image_config_digest: str, + runner: Runner, + sleeper: Callable[[float], None], +) -> None: + for _attempt in range(4): + completed = _docker(["container", "rm", container_id], runner) + if completed.returncode == 0: + _wait_absent(container_id, runner, sleeper) + return + observed = _inspect(container_id, runner) + if observed is None: + return + running = _validate_container( + observed, + container_id=container_id, + release_sha256=release_sha256, + image_reference=image_reference, + image_config_digest=image_config_digest, + ) + if running: + _stop_exact(container_id, runner, sleeper) + return + sleeper(0.25) + raise ControlError("created container removal failed") + + +def _complete_cidfile( + cidfile: Path, + identity: tuple[int, int], + *, + sleeper: Callable[[float], None], +) -> tuple[str, tuple[int, int]]: + for _attempt in range(40): + sleeper(0.25) + record = _read_cidfile(cidfile) + _require(record is not None and record[1] == identity, "container ID file changed during startup") + if record[0] is not None: + return record[0], record[1] + raise ControlError("container ID file did not become complete within the bounded wait") + + def control_container( action: str, release_sha256: str, @@ -253,6 +302,13 @@ def control_container( return {"action": action, "status": "pass", "container": "absent"} container_id, cid_identity = cid + if container_id is None: + container_id, cid_identity = _complete_cidfile( + cidfile, + cid_identity, + sleeper=sleeper, + ) + by_name = _inspect(CONTAINER_NAME, runner) by_id = _inspect(container_id, runner) if by_id is None: _require(by_name is None, "container name was rebound after the trusted container exited") @@ -274,7 +330,18 @@ def control_container( if running: _stop_exact(container_id, runner, sleeper) else: - _wait_absent(container_id, runner, sleeper) + _require( + isinstance(by_id.get("State"), dict) and by_id["State"].get("Status") == "created", + "non-running container was not safely removable", + ) + _remove_created_exact( + container_id, + release_sha256=release_sha256, + image_reference=image_reference, + image_config_digest=image_config_digest, + runner=runner, + sleeper=sleeper, + ) _require_name_unbound(runner) _unlink_cidfile(cidfile, cid_identity) return {"action": action, "status": "pass", "container": "stopped"} diff --git a/tests/test_gcp_leoclean_nosend_installer.py b/tests/test_gcp_leoclean_nosend_installer.py index b2039ab..e2bd5dc 100644 --- a/tests/test_gcp_leoclean_nosend_installer.py +++ b/tests/test_gcp_leoclean_nosend_installer.py @@ -189,7 +189,12 @@ class FakeHost: "Env": runtime["environment"], "Labels": labels, }, - "State": {"Running": True, "Pid": 2000 + self.generation, "Health": {"Status": "healthy"}}, + "State": { + "Status": "running", + "Running": True, + "Pid": 2000 + self.generation, + "Health": {"Status": "healthy"}, + }, "HostConfig": { "ReadonlyRootfs": True, "NetworkMode": "bridge", @@ -199,6 +204,10 @@ class FakeHost: "SecurityOpt": ["no-new-privileges:true"], }, } + self.paths.cidfile.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + self.paths.cidfile.parent.chmod(0o700) + self.paths.cidfile.write_text(container_id, encoding="ascii") + self.paths.cidfile.chmod(0o600) self.current_release = release self._write_process_environment() @@ -209,6 +218,11 @@ class FakeHost: self.invocation = "" self.container = None self.current_release = None + self.paths.cidfile.unlink(missing_ok=True) + try: + self.paths.cidfile.parent.rmdir() + except FileNotFoundError: + pass def _show(self, arguments: list[str]) -> subprocess.CompletedProcess[bytes]: self.show_count += 1 @@ -279,7 +293,8 @@ class FakeHost: return subprocess.CompletedProcess(arguments, 0, stdout=json.dumps([image]).encode(), stderr=b"") if operation[:2] == ["container", "inspect"]: if self.container is None or operation[2] not in {package.CONTAINER_NAME, self.container["Id"]}: - return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=b"not found") + error = f"Error: No such container: {operation[2]}\n".encode() + return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=error) observed = copy.deepcopy(self.container) release_sha = self.current_release.get("release_sha256") if self.current_release else None if operation[2] != package.CONTAINER_NAME and release_sha == self.race_release and not self.race_applied: @@ -455,6 +470,136 @@ def test_first_install_accepts_production_shaped_systemd_not_found_defaults(tmp_ assert host.restart_calls == 1 +def test_first_install_waits_for_complete_cid_and_healthy_container(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + container_inspections = 0 + completed_cid = False + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + nonlocal container_inspections + result = host(arguments) + if arguments[:2] == [installer.SYSTEMCTL, "restart"]: + assert host.container is not None + paths.cidfile.write_text(str(host.container["Id"])[:12], encoding="ascii") + paths.cidfile.chmod(0o600) + operation = arguments[3:] + if operation == ["container", "inspect", package.CONTAINER_NAME] and result.returncode == 0: + container_inspections += 1 + if container_inspections == 1: + observed = json.loads(result.stdout) + observed[0]["State"]["Health"]["Status"] = "starting" + return subprocess.CompletedProcess( + arguments, + 0, + stdout=json.dumps(observed).encode(), + stderr=b"", + ) + return result + + def sleeper(_seconds: float) -> None: + nonlocal completed_cid + if host.active and not completed_cid: + assert host.container is not None + paths.cidfile.write_text(str(host.container["Id"]), encoding="ascii") + paths.cidfile.chmod(0o600) + completed_cid = True + + receipt = installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=runner, + sleeper=sleeper, + execute_restart=True, + ) + + assert receipt["status"] == "pass" + assert completed_cid + assert container_inspections == 2 + assert host.stop_calls == 0 + + +def test_first_install_health_timeout_rolls_back_to_absent(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + result = host(arguments) + if arguments[3:] == ["container", "inspect", package.CONTAINER_NAME] and result.returncode == 0: + observed = json.loads(result.stdout) + observed[0]["State"]["Health"]["Status"] = "starting" + return subprocess.CompletedProcess( + arguments, + 0, + stdout=json.dumps(observed).encode(), + stderr=b"", + ) + return result + + with pytest.raises(installer.InstallError, match="did not become healthy"): + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=runner, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + + assert host.stop_calls == 1 + assert not host.active + assert host.container is None + assert not paths.cidfile.exists() + assert not paths.unit.exists() + assert not paths.helper.exists() + assert not paths.state_dir.exists() + + +def test_verification_rebinds_stable_container_to_unchanged_cidfile(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + changed = False + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + nonlocal changed + result = host(arguments) + operation = arguments[3:] + if ( + operation[:2] == ["container", "inspect"] + and operation[2] != package.CONTAINER_NAME + and not changed + ): + paths.cidfile.write_text("f" * 64, encoding="ascii") + paths.cidfile.chmod(0o600) + changed = True + return result + + with pytest.raises(installer.InstallError, match="changed during verification"): + installer.verify_running_release( + release, + runner=runner, + paths=paths, + sleeper=lambda _seconds: None, + ) + + assert changed + + def test_systemd_252_omitted_environment_files_property_is_normalized(tmp_path: Path) -> None: bundle, release, image = release_bundle(tmp_path, "2") paths = install_paths(tmp_path) @@ -992,6 +1137,67 @@ def test_restart_command_failure_after_side_effect_stops_candidate_and_cleans_in assert not paths.state_dir.exists() +def test_candidate_stop_failure_preserves_recovery_artifacts(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + host.race_release = str(release["release_sha256"]) + host.race_kind = "service_restart" + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + if arguments[:2] == [installer.SYSTEMCTL, "stop"]: + return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=b"stop failed") + return host(arguments) + + with pytest.raises(installer.InstallError, match="automatic rollback was incomplete"): + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=runner, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + + assert paths.unit.is_file() + assert paths.helper.is_file() + assert paths.state_dir.is_dir() + assert host.active + assert host.container is not None + + +def test_successful_stop_command_with_candidate_residue_preserves_recovery_artifacts(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + host.race_release = str(release["release_sha256"]) + host.race_kind = "service_restart" + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + if arguments[:2] == [installer.SYSTEMCTL, "stop"]: + return subprocess.CompletedProcess(arguments, 0, stdout=b"", stderr=b"") + return host(arguments) + + with pytest.raises(installer.InstallError, match="automatic rollback was incomplete"): + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=runner, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + + assert paths.unit.is_file() + assert paths.helper.is_file() + assert paths.state_dir.is_dir() + assert paths.cidfile.is_file() + assert host.active + assert host.container is not None + + def test_later_dropin_failure_restores_previous_release_and_running_service(tmp_path: Path) -> None: first_bundle, first_release, first_image = release_bundle(tmp_path, "2") second_bundle, second_release, second_image = release_bundle(tmp_path, "4") @@ -1481,6 +1687,187 @@ def test_control_helper_stops_only_trusted_full_container_id(tmp_path: Path) -> assert stop_calls == [["container", "stop", "--time=30", container_id]] +@pytest.mark.parametrize("running", [True, False]) +def test_control_helper_waits_for_complete_cid_before_exact_cleanup( + tmp_path: Path, + running: bool, +) -> None: + cid_parent = tmp_path / "runtime" + cid_parent.mkdir(mode=0o700) + cidfile = cid_parent / "container.cid" + container_id = "c" * 64 + cidfile.write_text(container_id[:12], encoding="ascii") + cidfile.chmod(0o600) + release_sha = "d" * 64 + image_reference = f"{package.IMAGE_REPOSITORY}@sha256:{'e' * 64}" + pending_container: dict[str, object] = { + "Id": container_id, + "Name": f"/{package.CONTAINER_NAME}", + "Image": "sha256:" + "f" * 64, + "Config": { + "Image": image_reference, + "Labels": { + "livingip.release.sha256": release_sha, + "livingip.container.role": "leoclean-gcp-nosend", + }, + }, + "State": { + "Status": "running" if running else "created", + "Running": running, + }, + } + container: dict[str, object] | None = None + calls: list[list[str]] = [] + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + nonlocal container + calls.append(arguments) + operation = arguments[3:] + if operation[:2] == ["container", "inspect"]: + if container is None: + error = f"Error: No such container: {operation[2]}\n".encode() + return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=error) + return subprocess.CompletedProcess(arguments, 0, stdout=json.dumps([container]).encode(), stderr=b"") + expected_mutation = ( + ["container", "stop", "--time=30", container_id] + if running + else ["container", "rm", container_id] + ) + if operation == expected_mutation: + container = None + return subprocess.CompletedProcess(arguments, 0, stdout=b"", stderr=b"") + raise AssertionError(arguments) + + completed = False + + def sleeper(_seconds: float) -> None: + nonlocal completed, container + if not completed: + cidfile.write_text(container_id, encoding="ascii") + cidfile.chmod(0o600) + container = pending_container + completed = True + + result = control.control_container( + "stop", + release_sha, + image_reference, + "sha256:" + "f" * 64, + runner=runner, + sleeper=sleeper, + cidfile=cidfile, + ) + + assert result == {"action": "stop", "status": "pass", "container": "stopped"} + assert completed + assert not cidfile.exists() + mutations = [ + call[3:] + for call in calls + if call[3:5] in (["container", "stop"], ["container", "rm"]) + ] + assert mutations == [ + ["container", "stop", "--time=30", container_id] + if running + else ["container", "rm", container_id] + ] + + +def test_control_helper_reconciles_created_to_running_race_by_full_id(tmp_path: Path) -> None: + cid_parent = tmp_path / "runtime" + cid_parent.mkdir(mode=0o700) + cidfile = cid_parent / "container.cid" + container_id = "c" * 64 + cidfile.write_text(container_id, encoding="ascii") + cidfile.chmod(0o600) + release_sha = "d" * 64 + image_reference = f"{package.IMAGE_REPOSITORY}@sha256:{'e' * 64}" + container: dict[str, object] | None = { + "Id": container_id, + "Name": f"/{package.CONTAINER_NAME}", + "Image": "sha256:" + "f" * 64, + "Config": { + "Image": image_reference, + "Labels": { + "livingip.release.sha256": release_sha, + "livingip.container.role": "leoclean-gcp-nosend", + }, + }, + "State": {"Status": "created", "Running": False}, + } + calls: list[list[str]] = [] + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + nonlocal container + calls.append(arguments) + operation = arguments[3:] + if operation[:2] == ["container", "inspect"]: + if container is None: + error = f"Error: No such container: {operation[2]}\n".encode() + return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=error) + return subprocess.CompletedProcess(arguments, 0, stdout=json.dumps([container]).encode(), stderr=b"") + if operation == ["container", "rm", container_id]: + assert container is not None + container["State"] = {"Status": "running", "Running": True} + return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=b"started during removal") + if operation == ["container", "stop", "--time=30", container_id]: + container = None + return subprocess.CompletedProcess(arguments, 0, stdout=b"", stderr=b"") + raise AssertionError(arguments) + + result = control.control_container( + "stop", + release_sha, + image_reference, + "sha256:" + "f" * 64, + runner=runner, + sleeper=lambda _seconds: None, + cidfile=cidfile, + ) + + assert result == {"action": "stop", "status": "pass", "container": "stopped"} + assert not cidfile.exists() + mutations = [ + call[3:] + for call in calls + if call[3:5] in (["container", "stop"], ["container", "rm"]) + ] + assert mutations == [ + ["container", "rm", container_id], + ["container", "stop", "--time=30", container_id], + ] + + +def test_control_helper_partial_cid_timeout_retains_authority_and_does_not_mutate(tmp_path: Path) -> None: + cid_parent = tmp_path / "runtime" + cid_parent.mkdir(mode=0o700) + cidfile = cid_parent / "container.cid" + cidfile.write_text("c" * 12, encoding="ascii") + cidfile.chmod(0o600) + calls: list[list[str]] = [] + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + calls.append(arguments) + operation = arguments[3:] + assert operation == ["container", "inspect", package.CONTAINER_NAME] + error = f"Error: No such container: {package.CONTAINER_NAME}\n".encode() + return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=error) + + with pytest.raises(control.ControlError, match="did not become complete"): + control.control_container( + "stop", + "d" * 64, + f"{package.IMAGE_REPOSITORY}@sha256:{'e' * 64}", + "sha256:" + "f" * 64, + runner=runner, + sleeper=lambda _seconds: None, + cidfile=cidfile, + ) + + assert cidfile.read_text(encoding="ascii") == "c" * 12 + assert all(call[3:5] not in (["container", "stop"], ["container", "rm"]) for call in calls) + + def test_control_helper_refuses_config_digest_mismatch_without_stopping(tmp_path: Path) -> None: cid_parent = tmp_path / "runtime" cid_parent.mkdir(mode=0o700)