From 4df77c7c6e340bdc78494081a2ec0ae9b3759cfa Mon Sep 17 00:00:00 2001 From: fwazb Date: Thu, 23 Jul 2026 13:01:55 -0700 Subject: [PATCH] Expose bounded container contract failures --- ops/install_gcp_leoclean_nosend_release.py | 192 +++++++++++--- tests/test_gcp_leoclean_nosend_installer.py | 264 ++++++++++++++++++-- 2 files changed, 399 insertions(+), 57 deletions(-) diff --git a/ops/install_gcp_leoclean_nosend_release.py b/ops/install_gcp_leoclean_nosend_release.py index 8f7740e..6f5c31e 100644 --- a/ops/install_gcp_leoclean_nosend_release.py +++ b/ops/install_gcp_leoclean_nosend_release.py @@ -121,7 +121,31 @@ VERIFICATION_FAILURES = frozenset( ("startup_identity", "container_id_invalid"), ("process_environment", "process_environment_mismatch"), ("supervising_process", "supervising_process_mismatch"), - ("container_startup", "container_contract_mismatch"), + ("container_inspection", "container_inspection_invalid"), + ("container_inspection", "container_config_invalid"), + ("container_inspection", "container_state_invalid"), + ("container_inspection", "container_host_config_invalid"), + ("container_identity", "container_id_mismatch"), + ("container_identity", "container_name_mismatch"), + ("container_image_identity", "container_image_id_mismatch"), + ("container_image_identity", "container_image_reference_mismatch"), + ("container_environment", "container_environment_mismatch"), + ("container_labels", "container_labels_invalid"), + ("container_labels", "container_release_label_mismatch"), + ("container_labels", "container_role_label_mismatch"), + ("container_security", "container_readonly_rootfs_mismatch"), + ("container_security", "container_network_mode_mismatch"), + ("container_security", "container_pids_limit_mismatch"), + ("container_security", "container_cap_drop_mismatch"), + ("container_security", "container_cap_add_mismatch"), + ("container_security", "container_security_options_mismatch"), + ("container_command", "container_path_mismatch"), + ("container_command", "container_args_mismatch"), + ("container_state", "container_running_flag_invalid"), + ("container_state", "container_pid_invalid"), + ("container_state", "container_exited_early"), + ("container_health", "container_health_missing"), + ("container_health", "container_health_status_invalid"), ("container_startup", "container_not_healthy"), ("image_identity", "image_contract_mismatch"), ("stable_container", "container_stability_mismatch"), @@ -1051,48 +1075,148 @@ def _validate_starting_container( ) -> dict[str, Any] | None: config = value.get("Config") state = value.get("State") - health = state.get("Health") if isinstance(state, dict) else None host = value.get("HostConfig") - labels = config.get("Labels") if isinstance(config, dict) else None runtime = release["inspection"]["runtime_config"] container_id = value.get("Id") - _require( - container_id == expected_container_id - and HEX_64.fullmatch(expected_container_id) is not None - and value.get("Name") == f"/{package.CONTAINER_NAME}" - and value.get("Image") == release["image"]["config_digest"] - and isinstance(config, dict) - and config.get("Image") == release["image"]["reference"] - and config.get("Env") == runtime["environment"] - and isinstance(labels, dict) - and labels.get("livingip.release.sha256") == release["release_sha256"] - and labels.get("livingip.container.role") == "leoclean-gcp-nosend" - and isinstance(state, dict) - and isinstance(host, dict) - and host.get("ReadonlyRootfs") is True - and host.get("NetworkMode") == "bridge" - and host.get("PidsLimit") == 512 - and sorted(host.get("CapDrop", [])) == ["ALL"] - and sorted(host.get("CapAdd", [])) == ["CHOWN", "SETGID", "SETPCAP", "SETUID"] - and host.get("SecurityOpt") == ["no-new-privileges:true"], - "running container differed from the release contract", + _verification_require( + container_id == expected_container_id and HEX_64.fullmatch(expected_container_id) is not None, + "container_identity", + "container_id_mismatch", + ) + _verification_require( + value.get("Name") == f"/{package.CONTAINER_NAME}", + "container_identity", + "container_name_mismatch", + ) + _verification_require( + value.get("Image") == release["image"]["config_digest"], + "container_image_identity", + "container_image_id_mismatch", + ) + _verification_require( + isinstance(config, dict), + "container_inspection", + "container_config_invalid", + ) + _verification_require( + config.get("Image") == release["image"]["reference"], + "container_image_identity", + "container_image_reference_mismatch", + ) + _verification_require( + config.get("Env") == runtime["environment"], + "container_environment", + "container_environment_mismatch", + ) + labels = config.get("Labels") + _verification_require( + isinstance(labels, dict), + "container_labels", + "container_labels_invalid", + ) + _verification_require( + labels.get("livingip.release.sha256") == release["release_sha256"], + "container_labels", + "container_release_label_mismatch", + ) + _verification_require( + labels.get("livingip.container.role") == "leoclean-gcp-nosend", + "container_labels", + "container_role_label_mismatch", + ) + _verification_require( + isinstance(state, dict), + "container_inspection", + "container_state_invalid", + ) + _verification_require( + isinstance(host, dict), + "container_inspection", + "container_host_config_invalid", + ) + _verification_require( + host.get("ReadonlyRootfs") is True, + "container_security", + "container_readonly_rootfs_mismatch", + ) + _verification_require( + host.get("NetworkMode") == "bridge", + "container_security", + "container_network_mode_mismatch", + ) + _verification_require( + host.get("PidsLimit") == 512, + "container_security", + "container_pids_limit_mismatch", + ) + _verification_require( + isinstance(host.get("CapDrop"), list) + and all(isinstance(item, str) for item in host["CapDrop"]) + and sorted(host["CapDrop"]) == ["ALL"], + "container_security", + "container_cap_drop_mismatch", + ) + _verification_require( + isinstance(host.get("CapAdd"), list) + and all(isinstance(item, str) for item in host["CapAdd"]) + and sorted(host["CapAdd"]) == ["CHOWN", "SETGID", "SETPCAP", "SETUID"], + "container_security", + "container_cap_add_mismatch", + ) + _verification_require( + host.get("SecurityOpt") == ["no-new-privileges:true"], + "container_security", + "container_security_options_mismatch", ) expected_path = runtime["entrypoint"][0] expected_args = [*runtime["entrypoint"][1:], *runtime["cmd"]] - _require( - value.get("Path") == expected_path and value.get("Args") == expected_args, - "running container command differed from the release contract", + _verification_require( + value.get("Path") == expected_path, + "container_command", + "container_path_mismatch", + ) + _verification_require( + value.get("Args") == expected_args, + "container_command", + "container_args_mismatch", ) running = state.get("Running") pid = state.get("Pid") status = state.get("Status") - _require(isinstance(running, bool) and isinstance(pid, int), "container startup state was invalid") + _verification_require( + isinstance(running, bool), + "container_state", + "container_running_flag_invalid", + ) + _verification_require( + isinstance(pid, int), + "container_state", + "container_pid_invalid", + ) if running is False: - _require(status == "created" and pid == 0, "container exited before becoming healthy") + _verification_require( + status == "created" and pid == 0, + "container_state", + "container_exited_early", + ) return None - _require(pid > 1 and isinstance(health, dict), "container startup state was invalid") + _verification_require( + pid > 1, + "container_state", + "container_pid_invalid", + ) + health = state.get("Health") + _verification_require( + isinstance(health, dict), + "container_health", + "container_health_missing", + ) health_status = health.get("Status") - _require(health_status in {"starting", "healthy"}, "container became unhealthy during startup") + _verification_require( + health_status in {"starting", "healthy"}, + "container_health", + "container_health_status_invalid", + ) if health_status == "starting": return None return {"container_id": container_id, "container_pid": state["Pid"], "health": "healthy"} @@ -1195,8 +1319,8 @@ def verify_running_release( ), ) first_container = _verification_check( - "container_startup", - "container_contract_mismatch", + "container_inspection", + "container_inspection_invalid", lambda: _docker_inspect( ["container", "inspect", package.CONTAINER_NAME], runner, @@ -1208,8 +1332,8 @@ def verify_running_release( sleeper(0.25) continue container = _verification_check( - "container_startup", - "container_contract_mismatch", + "container_inspection", + "container_inspection_invalid", lambda first_container=first_container, container_id=container_id: _validate_starting_container( first_container, release, diff --git a/tests/test_gcp_leoclean_nosend_installer.py b/tests/test_gcp_leoclean_nosend_installer.py index 55c762f..d100c50 100644 --- a/tests/test_gcp_leoclean_nosend_installer.py +++ b/tests/test_gcp_leoclean_nosend_installer.py @@ -255,8 +255,7 @@ class FakeHost: payload = "".join( f"{name}={value[name]}\n" for name in installer.SERVICE_PROPERTIES - if name not in self.omitted_properties - and (loaded or name not in self.not_found_omitted_properties) + if name not in self.omitted_properties and (loaded or name not in self.not_found_omitted_properties) ).encode() return subprocess.CompletedProcess(arguments, 0, stdout=payload, stderr=b"") @@ -594,6 +593,241 @@ def test_process_environment_failure_has_bounded_stage_and_reason(tmp_path: Path assert caught.value.failure_reason == "process_environment_mismatch" +@pytest.mark.parametrize( + ("path", "replacement", "failure_stage", "failure_reason"), + [ + (("Id",), "f" * 64, "container_identity", "container_id_mismatch"), + (("Name",), "/unexpected", "container_identity", "container_name_mismatch"), + ( + ("Image",), + f"sha256:{'f' * 64}", + "container_image_identity", + "container_image_id_mismatch", + ), + ( + ("Config", "Image"), + "example.invalid/unexpected@sha256:" + "f" * 64, + "container_image_identity", + "container_image_reference_mismatch", + ), + ( + ("Config", "Env"), + ["PATH=/unexpected"], + "container_environment", + "container_environment_mismatch", + ), + (("Config", "Labels"), [], "container_labels", "container_labels_invalid"), + ( + ("Config", "Labels", "livingip.release.sha256"), + "f" * 64, + "container_labels", + "container_release_label_mismatch", + ), + ( + ("Config", "Labels", "livingip.container.role"), + "unexpected", + "container_labels", + "container_role_label_mismatch", + ), + (("Config",), None, "container_inspection", "container_config_invalid"), + (("State",), None, "container_inspection", "container_state_invalid"), + (("HostConfig",), None, "container_inspection", "container_host_config_invalid"), + ( + ("HostConfig", "ReadonlyRootfs"), + False, + "container_security", + "container_readonly_rootfs_mismatch", + ), + ( + ("HostConfig", "NetworkMode"), + "host", + "container_security", + "container_network_mode_mismatch", + ), + ( + ("HostConfig", "PidsLimit"), + 513, + "container_security", + "container_pids_limit_mismatch", + ), + ( + ("HostConfig", "CapDrop"), + [], + "container_security", + "container_cap_drop_mismatch", + ), + ( + ("HostConfig", "CapAdd"), + [], + "container_security", + "container_cap_add_mismatch", + ), + ( + ("HostConfig", "SecurityOpt"), + ["no-new-privileges=true"], + "container_security", + "container_security_options_mismatch", + ), + (("Path",), "/unexpected", "container_command", "container_path_mismatch"), + (("Args",), ["unexpected"], "container_command", "container_args_mismatch"), + ( + ("State", "Running"), + "true", + "container_state", + "container_running_flag_invalid", + ), + (("State", "Pid"), "2001", "container_state", "container_pid_invalid"), + (("State", "Health"), None, "container_health", "container_health_missing"), + ( + ("State", "Health", "Status"), + "unhealthy", + "container_health", + "container_health_status_invalid", + ), + ], +) +def test_container_contract_failures_have_field_specific_bounded_codes( + tmp_path: Path, + path: tuple[str, ...], + replacement: object, + failure_stage: str, + failure_reason: str, +) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + assert host.container is not None + target = host.container + for key in path[:-1]: + nested = target[key] + assert isinstance(nested, dict) + target = nested + target[path[-1]] = copy.deepcopy(replacement) + + with pytest.raises(installer.VerificationError) as caught: + installer.verify_running_release( + release, + runner=host, + paths=paths, + sleeper=lambda _seconds: None, + ) + + assert caught.value.failure_stage == failure_stage + assert caught.value.failure_reason == failure_reason + + +def test_container_inspection_shape_has_bounded_code(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + result = host(arguments) + if arguments[3:] == ["container", "inspect", package.CONTAINER_NAME] and result.returncode == 0: + return subprocess.CompletedProcess(arguments, 0, stdout=b"{}", stderr=b"") + return result + + with pytest.raises(installer.VerificationError) as caught: + installer.verify_running_release( + release, + runner=runner, + paths=paths, + sleeper=lambda _seconds: None, + ) + + assert caught.value.failure_stage == "container_inspection" + assert caught.value.failure_reason == "container_inspection_invalid" + + +def test_container_exit_before_health_has_bounded_code(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + assert host.container is not None + state = host.container["State"] + assert isinstance(state, dict) + state.update({"Running": False, "Pid": 0, "Status": "exited"}) + + with pytest.raises(installer.VerificationError) as caught: + installer.verify_running_release( + release, + runner=host, + paths=paths, + sleeper=lambda _seconds: None, + ) + + assert caught.value.failure_stage == "container_state" + assert caught.value.failure_reason == "container_exited_early" + + +def test_container_security_mismatch_rolls_back_without_exposing_observed_value(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + result = host(arguments) + if arguments[3:] == ["container", "inspect", package.CONTAINER_NAME] and result.returncode == 0: + observed = json.loads(result.stdout) + observed[0]["HostConfig"]["SecurityOpt"] = ["unexpected-sensitive-value"] + return subprocess.CompletedProcess( + arguments, + 0, + stdout=json.dumps(observed).encode(), + stderr=b"", + ) + return result + + with pytest.raises(installer.VerificationError) as caught: + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=runner, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + + assert caught.value.failure_stage == "container_security" + assert caught.value.failure_reason == "container_security_options_mismatch" + assert "unexpected-sensitive-value" not in str(caught.value) + assert host.stop_calls == 1 + assert not host.active + assert host.container is None + assert not paths.cidfile.exists() + assert not paths.unit.exists() + assert not paths.helper.exists() + assert not paths.state_dir.exists() + + def test_first_install_health_timeout_rolls_back_to_absent(tmp_path: Path) -> None: bundle, release, image = release_bundle(tmp_path, "2") paths = install_paths(tmp_path) @@ -651,11 +885,7 @@ def test_verification_rebinds_stable_container_to_unchanged_cidfile(tmp_path: Pa nonlocal changed result = host(arguments) operation = arguments[3:] - if ( - operation[:2] == ["container", "inspect"] - and operation[2] != package.CONTAINER_NAME - and not changed - ): + if operation[:2] == ["container", "inspect"] and operation[2] != package.CONTAINER_NAME and not changed: paths.cidfile.write_text("f" * 64, encoding="ascii") paths.cidfile.chmod(0o600) changed = True @@ -1801,9 +2031,7 @@ def test_control_helper_waits_for_complete_cid_before_exact_cleanup( return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=error) return subprocess.CompletedProcess(arguments, 0, stdout=json.dumps([container]).encode(), stderr=b"") expected_mutation = ( - ["container", "stop", "--time=30", container_id] - if running - else ["container", "rm", container_id] + ["container", "stop", "--time=30", container_id] if running else ["container", "rm", container_id] ) if operation == expected_mutation: container = None @@ -1833,15 +2061,9 @@ def test_control_helper_waits_for_complete_cid_before_exact_cleanup( assert result == {"action": "stop", "status": "pass", "container": "stopped"} assert completed assert not cidfile.exists() - mutations = [ - call[3:] - for call in calls - if call[3:5] in (["container", "stop"], ["container", "rm"]) - ] + mutations = [call[3:] for call in calls if call[3:5] in (["container", "stop"], ["container", "rm"])] assert mutations == [ - ["container", "stop", "--time=30", container_id] - if running - else ["container", "rm", container_id] + ["container", "stop", "--time=30", container_id] if running else ["container", "rm", container_id] ] @@ -1899,11 +2121,7 @@ def test_control_helper_reconciles_created_to_running_race_by_full_id(tmp_path: assert result == {"action": "stop", "status": "pass", "container": "stopped"} assert not cidfile.exists() - mutations = [ - call[3:] - for call in calls - if call[3:5] in (["container", "stop"], ["container", "rm"]) - ] + mutations = [call[3:] for call in calls if call[3:5] in (["container", "stop"], ["container", "rm"])] assert mutations == [ ["container", "rm", container_id], ["container", "stop", "--time=30", container_id],