Merge pull request #242 from living-ip/fix/leoclean-install-convergence
Some checks are pending
CI / lint-and-test (push) Waiting to run
Some checks are pending
CI / lint-and-test (push) Waiting to run
Converge leoclean no-send installation state
This commit is contained in:
commit
f3a7650a2d
6 changed files with 2268 additions and 139 deletions
|
|
@ -324,9 +324,22 @@ disappears it rechecks that the canonical name is still unbound before removing
|
||||||
the CID file. It never removes a container, uses a mutable name as stop
|
the CID file. It never removes a container, uses a mutable name as stop
|
||||||
authority, or invokes `--force`.
|
authority, or invokes `--force`.
|
||||||
|
|
||||||
An already installed release is verified without a restart. During an upgrade,
|
Install is convergent: it means ensuring the requested exact release is
|
||||||
the installer accepts only a stable two-readback active/running or inactive/dead
|
installed, active, and verified. An exact active release is verified as a
|
||||||
prior posture. Before trusting the old `ExecStop`, it exact-binds effective
|
no-op. An exact inactive release receives one controlled `start`, while a
|
||||||
|
missing or different release uses the replacement transaction. The persisted
|
||||||
|
install history is not rewritten by exact-release convergence. New replacement
|
||||||
|
history uses `livingip.leocleanNoSendInstallReceipt.v3`; existing
|
||||||
|
`livingip.leocleanNoSendInstallReceipt.v2` history remains readable. Each
|
||||||
|
invocation returns a separate
|
||||||
|
`livingip.leocleanNoSendInstallExecution.v1` result whose transition records
|
||||||
|
`verified_noop`, `controlled_start`, `installed`, or `replaced`, the prior
|
||||||
|
posture and restart counter, the service action, and whether managed files were
|
||||||
|
replaced.
|
||||||
|
|
||||||
|
During an upgrade, the installer accepts only a stable two-readback
|
||||||
|
active/running or inactive/dead prior posture. Before trusting the old
|
||||||
|
`ExecStop`, it exact-binds effective
|
||||||
`ExecStartPre`, `ExecStart`, `ExecStop`, runtime-directory and systemd security
|
`ExecStartPre`, `ExecStart`, `ExecStop`, runtime-directory and systemd security
|
||||||
properties to the release and verifies the supervising PID's executable,
|
properties to the release and verifies the supervising PID's executable,
|
||||||
command line and cgroup through `/proc`. It then stops the prior service through
|
command line and cgroup through `/proc`. It then stops the prior service through
|
||||||
|
|
@ -339,6 +352,13 @@ two readbacks. On failure it stops the candidate, restores the exact prior
|
||||||
files, modes and active/inactive/absent posture, and reloads systemd. An
|
files, modes and active/inactive/absent posture, and reloads systemd. An
|
||||||
incomplete rollback fails closed and is reported as such.
|
incomplete rollback fails closed and is reported as such.
|
||||||
|
|
||||||
|
The IAP result and bounded failure contracts use the v2 result schemas. A
|
||||||
|
post-install failure restores the transition's proven prior posture without
|
||||||
|
consuming historical rollback for an exact-release no-op or controlled start.
|
||||||
|
Replacement failures use only the receipt-bound explicit rollback. Malformed
|
||||||
|
or missing transition data forbids historical rollback and instead contains
|
||||||
|
the service fail closed with a sanitized failure stage and reason.
|
||||||
|
|
||||||
This installer does not enable the service, provision IAM, roles or secrets,
|
This installer does not enable the service, provision IAM, roles or secrets,
|
||||||
contact GCP or Cloud SQL, change Telegram traffic, touch the VPS, approve a
|
contact GCP or Cloud SQL, change Telegram traffic, touch the VPS, approve a
|
||||||
proposal, or promote production. Those remain later, separately authorized
|
proposal, or promote production. Those remain later, separately authorized
|
||||||
|
|
|
||||||
|
|
@ -23,9 +23,9 @@ from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
SCHEMA = "livingip.leocleanNoSendIapRequest.v1"
|
SCHEMA = "livingip.leocleanNoSendIapRequest.v1"
|
||||||
RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v1"
|
RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v2"
|
||||||
FAILURE_RESULT_SCHEMA = "livingip.leocleanNoSendIapFailure.v1"
|
FAILURE_RESULT_SCHEMA = "livingip.leocleanNoSendIapFailure.v2"
|
||||||
OPERATION_FAILURE_RESULT_SCHEMA = "livingip.leocleanNoSendIapOperationalFailure.v1"
|
OPERATION_FAILURE_RESULT_SCHEMA = "livingip.leocleanNoSendIapOperationalFailure.v2"
|
||||||
PROJECT = "teleo-501523"
|
PROJECT = "teleo-501523"
|
||||||
ZONE = "europe-west6-a"
|
ZONE = "europe-west6-a"
|
||||||
INSTANCE = "teleo-staging-1"
|
INSTANCE = "teleo-staging-1"
|
||||||
|
|
@ -99,6 +99,164 @@ class RemoteOperationFailure(RemoteError):
|
||||||
self.result = dict(result)
|
self.result = dict(result)
|
||||||
|
|
||||||
|
|
||||||
|
class PostureRecoveryFailure(RemoteError):
|
||||||
|
"""A posture recovery failed after a bounded amount of service action."""
|
||||||
|
|
||||||
|
def __init__(self, *, executed: bool) -> None:
|
||||||
|
super().__init__("posture_recovery_failed")
|
||||||
|
self.executed = executed
|
||||||
|
|
||||||
|
|
||||||
|
class ExplicitRollbackFailure(RemoteError):
|
||||||
|
"""An explicit rollback failed before or after entering its mutation phase."""
|
||||||
|
|
||||||
|
def __init__(self, *, executed: bool) -> None:
|
||||||
|
super().__init__("explicit_rollback_failed")
|
||||||
|
self.executed = executed
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_installer_transition(value: Any) -> dict[str, Any]:
|
||||||
|
_require(
|
||||||
|
isinstance(value, dict)
|
||||||
|
and set(value)
|
||||||
|
== {
|
||||||
|
"outcome",
|
||||||
|
"prior_posture",
|
||||||
|
"prior_n_restarts",
|
||||||
|
"final_posture",
|
||||||
|
"service_action",
|
||||||
|
"managed_files_replaced",
|
||||||
|
},
|
||||||
|
"installer transition was invalid",
|
||||||
|
)
|
||||||
|
outcome = value.get("outcome")
|
||||||
|
prior_posture = value.get("prior_posture")
|
||||||
|
prior_n_restarts = value.get("prior_n_restarts")
|
||||||
|
service_action = value.get("service_action")
|
||||||
|
replaced = value.get("managed_files_replaced")
|
||||||
|
_require(
|
||||||
|
isinstance(prior_n_restarts, int)
|
||||||
|
and not isinstance(prior_n_restarts, bool)
|
||||||
|
and prior_n_restarts >= 0
|
||||||
|
and value.get("final_posture") == "active"
|
||||||
|
and (
|
||||||
|
(outcome, prior_posture, service_action, replaced)
|
||||||
|
== ("verified_noop", "active", "none", False)
|
||||||
|
or (outcome, prior_posture, service_action, replaced)
|
||||||
|
== ("controlled_start", "inactive", "start", False)
|
||||||
|
or (outcome, prior_posture, service_action, replaced)
|
||||||
|
== ("installed", "absent", "restart", True)
|
||||||
|
or (
|
||||||
|
outcome == "replaced"
|
||||||
|
and prior_posture in {"active", "inactive"}
|
||||||
|
and service_action == "restart"
|
||||||
|
and replaced is True
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"installer transition was invalid",
|
||||||
|
)
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_install_execution_result(
|
||||||
|
value: Any,
|
||||||
|
*,
|
||||||
|
release: Mapping[str, Any],
|
||||||
|
installer_module: Any,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
_require(
|
||||||
|
isinstance(value, dict)
|
||||||
|
and set(value)
|
||||||
|
== {
|
||||||
|
"schema",
|
||||||
|
"status",
|
||||||
|
"outcome",
|
||||||
|
"release_sha256",
|
||||||
|
"persisted_receipt",
|
||||||
|
"transition",
|
||||||
|
"verification",
|
||||||
|
},
|
||||||
|
"install execution result was invalid",
|
||||||
|
)
|
||||||
|
transition = _validate_installer_transition(value.get("transition"))
|
||||||
|
persisted_receipt = value.get("persisted_receipt")
|
||||||
|
outcome = value.get("outcome")
|
||||||
|
_require(
|
||||||
|
value.get("schema") == installer_module.INSTALL_EXECUTION_RESULT_SCHEMA
|
||||||
|
and value.get("status") == "pass"
|
||||||
|
and outcome == transition["outcome"]
|
||||||
|
and value.get("release_sha256") == release["release_sha256"]
|
||||||
|
and isinstance(persisted_receipt, dict)
|
||||||
|
and set(persisted_receipt) == {"schema", "sha256"}
|
||||||
|
and persisted_receipt.get("schema")
|
||||||
|
in {
|
||||||
|
installer_module.LEGACY_INSTALL_RECEIPT_SCHEMA,
|
||||||
|
installer_module.INSTALL_RECEIPT_SCHEMA,
|
||||||
|
}
|
||||||
|
and isinstance(persisted_receipt.get("sha256"), str)
|
||||||
|
and HEX_64.fullmatch(persisted_receipt["sha256"]) is not None
|
||||||
|
and isinstance(value.get("verification"), dict)
|
||||||
|
and (
|
||||||
|
outcome in {"verified_noop", "controlled_start"}
|
||||||
|
or (
|
||||||
|
outcome in {"installed", "replaced"}
|
||||||
|
and persisted_receipt["schema"] == installer_module.INSTALL_RECEIPT_SCHEMA
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"install execution result was invalid",
|
||||||
|
)
|
||||||
|
return transition
|
||||||
|
|
||||||
|
|
||||||
|
def _rollback_summary(
|
||||||
|
*,
|
||||||
|
status: str,
|
||||||
|
mode: str,
|
||||||
|
executed: bool,
|
||||||
|
restored_posture: str,
|
||||||
|
prior_posture_preserved: bool,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
value = {
|
||||||
|
"status": status,
|
||||||
|
"mode": mode,
|
||||||
|
"executed": executed,
|
||||||
|
"restored_posture": restored_posture,
|
||||||
|
"prior_posture_preserved": prior_posture_preserved,
|
||||||
|
}
|
||||||
|
_require(
|
||||||
|
(
|
||||||
|
status == "pass"
|
||||||
|
and (
|
||||||
|
(
|
||||||
|
mode in {"installer_automatic", "explicit"}
|
||||||
|
and executed is True
|
||||||
|
and restored_posture in {"active", "inactive", "absent"}
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
mode == "posture_recovery"
|
||||||
|
and restored_posture in {"active", "inactive"}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
and prior_posture_preserved is True
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
status == "not_run"
|
||||||
|
and mode == "not_applicable"
|
||||||
|
and executed is False
|
||||||
|
and restored_posture == "unchanged"
|
||||||
|
and prior_posture_preserved is True
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
status == "fail"
|
||||||
|
and mode in {"explicit", "posture_recovery"}
|
||||||
|
and restored_posture == "unknown"
|
||||||
|
and prior_posture_preserved is False
|
||||||
|
),
|
||||||
|
"rollback summary was invalid",
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
def _require(condition: bool, message: str) -> None:
|
def _require(condition: bool, message: str) -> None:
|
||||||
if not condition:
|
if not condition:
|
||||||
raise RemoteError(message)
|
raise RemoteError(message)
|
||||||
|
|
@ -716,20 +874,43 @@ def _execute_explicit_rollback(
|
||||||
|
|
||||||
rollback = getattr(installer_module, "rollback_release", None)
|
rollback = getattr(installer_module, "rollback_release", None)
|
||||||
_require(callable(rollback), "reviewed installer lacked the explicit rollback interface")
|
_require(callable(rollback), "reviewed installer lacked the explicit rollback interface")
|
||||||
result = rollback(
|
executed = False
|
||||||
release_sha256,
|
|
||||||
paths=paths,
|
def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]:
|
||||||
source_binding=source_binding,
|
return _run(
|
||||||
runner=lambda argv: _run(
|
|
||||||
argv,
|
argv,
|
||||||
runner=runner,
|
runner=runner,
|
||||||
label="rollback command",
|
label="rollback command",
|
||||||
require_success=False,
|
require_success=False,
|
||||||
),
|
)
|
||||||
execute_rollback=True,
|
|
||||||
)
|
try:
|
||||||
_require(isinstance(result, dict) and result.get("status") == "pass", "explicit rollback did not pass")
|
plan = rollback(
|
||||||
return result
|
release_sha256,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding,
|
||||||
|
runner=contract_runner,
|
||||||
|
execute_rollback=False,
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
isinstance(plan, dict)
|
||||||
|
and plan.get("status") == "pass"
|
||||||
|
and plan.get("mode") == "dry_run"
|
||||||
|
and plan.get("expected_release_sha256") == release_sha256,
|
||||||
|
"explicit rollback preflight did not pass",
|
||||||
|
)
|
||||||
|
executed = True
|
||||||
|
result = rollback(
|
||||||
|
release_sha256,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding,
|
||||||
|
runner=contract_runner,
|
||||||
|
execute_rollback=True,
|
||||||
|
)
|
||||||
|
_require(isinstance(result, dict) and result.get("status") == "pass", "explicit rollback did not pass")
|
||||||
|
return result
|
||||||
|
except Exception as exc:
|
||||||
|
raise ExplicitRollbackFailure(executed=executed) from exc
|
||||||
|
|
||||||
|
|
||||||
def _rollback_result(
|
def _rollback_result(
|
||||||
|
|
@ -808,7 +989,7 @@ def _verification_failure_result(
|
||||||
manifest: Mapping[str, Any],
|
manifest: Mapping[str, Any],
|
||||||
release: Mapping[str, Any],
|
release: Mapping[str, Any],
|
||||||
failure: Exception,
|
failure: Exception,
|
||||||
rollback: Mapping[str, str],
|
rollback: Mapping[str, Any],
|
||||||
archive_sha256: str,
|
archive_sha256: str,
|
||||||
helper_sha256: str,
|
helper_sha256: str,
|
||||||
installer_module: Any,
|
installer_module: Any,
|
||||||
|
|
@ -822,13 +1003,13 @@ def _verification_failure_result(
|
||||||
"verification failure was not allowlisted",
|
"verification failure was not allowlisted",
|
||||||
)
|
)
|
||||||
_require(
|
_require(
|
||||||
rollback
|
dict(rollback)
|
||||||
in (
|
== _rollback_summary(
|
||||||
{"status": "pass", "mode": "installer_automatic", "restored_posture": "prior"},
|
status=str(rollback.get("status")),
|
||||||
{"status": "pass", "mode": "explicit", "restored_posture": "active"},
|
mode=str(rollback.get("mode")),
|
||||||
{"status": "pass", "mode": "explicit", "restored_posture": "inactive"},
|
executed=rollback.get("executed") is True,
|
||||||
{"status": "pass", "mode": "explicit", "restored_posture": "absent"},
|
restored_posture=str(rollback.get("restored_posture")),
|
||||||
{"status": "not_run", "mode": "not_applicable", "restored_posture": "unchanged"},
|
prior_posture_preserved=rollback.get("prior_posture_preserved") is True,
|
||||||
),
|
),
|
||||||
"verification rollback summary was invalid",
|
"verification rollback summary was invalid",
|
||||||
)
|
)
|
||||||
|
|
@ -860,6 +1041,25 @@ def _verification_failure_result(
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _installer_failure_rollback(failure: Exception) -> dict[str, Any]:
|
||||||
|
rollback_executed = getattr(failure, "rollback_executed", None)
|
||||||
|
restored_posture = getattr(failure, "restored_posture", None)
|
||||||
|
prior_posture_preserved = getattr(failure, "prior_posture_preserved", None)
|
||||||
|
_require(
|
||||||
|
isinstance(rollback_executed, bool)
|
||||||
|
and restored_posture in {"active", "inactive", "absent"}
|
||||||
|
and prior_posture_preserved is True,
|
||||||
|
"installer failure lacked a proven recovery posture",
|
||||||
|
)
|
||||||
|
return _rollback_summary(
|
||||||
|
status="pass" if rollback_executed else "not_run",
|
||||||
|
mode="installer_automatic" if rollback_executed else "not_applicable",
|
||||||
|
executed=rollback_executed,
|
||||||
|
restored_posture=str(restored_posture) if rollback_executed else "unchanged",
|
||||||
|
prior_posture_preserved=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _is_verification_failure(installer_module: Any, failure: Exception) -> bool:
|
def _is_verification_failure(installer_module: Any, failure: Exception) -> bool:
|
||||||
failure_type = getattr(installer_module, "VerificationError", None)
|
failure_type = getattr(installer_module, "VerificationError", None)
|
||||||
return isinstance(failure_type, type) and isinstance(failure, failure_type)
|
return isinstance(failure_type, type) and isinstance(failure, failure_type)
|
||||||
|
|
@ -899,6 +1099,7 @@ def _rollback_proven_service(posture: str) -> dict[str, Any]:
|
||||||
"main_pid": None,
|
"main_pid": None,
|
||||||
"n_restarts": None,
|
"n_restarts": None,
|
||||||
"candidate_container": "not_observed",
|
"candidate_container": "not_observed",
|
||||||
|
"service_action_executed": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -909,6 +1110,8 @@ def _fail_closed_inert_service(
|
||||||
paths: Any,
|
paths: Any,
|
||||||
runner: Runner,
|
runner: Runner,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
|
service_action_executed = False
|
||||||
|
|
||||||
def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]:
|
def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]:
|
||||||
return _run(
|
return _run(
|
||||||
argv,
|
argv,
|
||||||
|
|
@ -949,6 +1152,7 @@ def _fail_closed_inert_service(
|
||||||
and verification.get("n_restarts") == before.state.get("NRestarts"),
|
and verification.get("n_restarts") == before.state.get("NRestarts"),
|
||||||
"running service identity changed before fail-closed stop",
|
"running service identity changed before fail-closed stop",
|
||||||
)
|
)
|
||||||
|
service_action_executed = True
|
||||||
systemctl("stop", contract_runner)
|
systemctl("stop", contract_runner)
|
||||||
after = stable_service_posture(
|
after = stable_service_posture(
|
||||||
release,
|
release,
|
||||||
|
|
@ -979,6 +1183,7 @@ def _fail_closed_inert_service(
|
||||||
"main_pid": 0,
|
"main_pid": 0,
|
||||||
"n_restarts": after.state["NRestarts"],
|
"n_restarts": after.state["NRestarts"],
|
||||||
"candidate_container": "absent",
|
"candidate_container": "absent",
|
||||||
|
"service_action_executed": service_action_executed,
|
||||||
}
|
}
|
||||||
except Exception:
|
except Exception:
|
||||||
return {
|
return {
|
||||||
|
|
@ -989,16 +1194,128 @@ def _fail_closed_inert_service(
|
||||||
"main_pid": None,
|
"main_pid": None,
|
||||||
"n_restarts": None,
|
"n_restarts": None,
|
||||||
"candidate_container": "unknown",
|
"candidate_container": "unknown",
|
||||||
|
"service_action_executed": service_action_executed,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _restore_transition_posture(
|
||||||
|
*,
|
||||||
|
installer_module: Any,
|
||||||
|
release: Mapping[str, Any],
|
||||||
|
transition: Mapping[str, Any],
|
||||||
|
paths: Any,
|
||||||
|
runner: Runner,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
checked = _validate_installer_transition(transition)
|
||||||
|
desired = str(checked["prior_posture"])
|
||||||
|
_require(
|
||||||
|
checked["outcome"] in {"verified_noop", "controlled_start"}
|
||||||
|
and desired in {"active", "inactive"},
|
||||||
|
"installer transition required release rollback",
|
||||||
|
)
|
||||||
|
|
||||||
|
def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]:
|
||||||
|
return _run(
|
||||||
|
argv,
|
||||||
|
runner=runner,
|
||||||
|
label="prior posture recovery command",
|
||||||
|
require_success=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
systemctl = getattr(installer_module, "_systemctl", None)
|
||||||
|
stable_service_posture = getattr(installer_module, "_stable_service_posture", None)
|
||||||
|
verify_running_release = getattr(installer_module, "verify_running_release", None)
|
||||||
|
require_candidate_absent = getattr(installer_module, "_require_candidate_absent", None)
|
||||||
|
_require(
|
||||||
|
callable(systemctl)
|
||||||
|
and callable(stable_service_posture)
|
||||||
|
and callable(verify_running_release)
|
||||||
|
and callable(require_candidate_absent),
|
||||||
|
"reviewed installer lacked posture recovery controls",
|
||||||
|
)
|
||||||
|
executed = False
|
||||||
|
try:
|
||||||
|
observed = stable_service_posture(
|
||||||
|
release,
|
||||||
|
runner=contract_runner,
|
||||||
|
paths=paths,
|
||||||
|
sleeper=time.sleep,
|
||||||
|
)
|
||||||
|
_require(observed.kind in {"active", "inactive"}, "managed service posture was not trusted")
|
||||||
|
if desired == "inactive":
|
||||||
|
if observed.kind == "active":
|
||||||
|
verification = verify_running_release(
|
||||||
|
release,
|
||||||
|
runner=contract_runner,
|
||||||
|
paths=paths,
|
||||||
|
sleeper=time.sleep,
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
verification.get("main_pid") == observed.state.get("MainPID")
|
||||||
|
and verification.get("invocation_id") == observed.state.get("InvocationID")
|
||||||
|
and verification.get("n_restarts") == observed.state.get("NRestarts"),
|
||||||
|
"running service identity changed before posture recovery stop",
|
||||||
|
)
|
||||||
|
executed = True
|
||||||
|
systemctl("stop", contract_runner)
|
||||||
|
final = stable_service_posture(
|
||||||
|
release,
|
||||||
|
runner=contract_runner,
|
||||||
|
paths=paths,
|
||||||
|
sleeper=time.sleep,
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
final.kind == "inactive"
|
||||||
|
and final.state.get("NRestarts") == checked["prior_n_restarts"],
|
||||||
|
"prior inactive posture was not restored",
|
||||||
|
)
|
||||||
|
require_candidate_absent(runner=contract_runner, paths=paths, sleeper=time.sleep)
|
||||||
|
else:
|
||||||
|
if observed.kind == "inactive":
|
||||||
|
require_candidate_absent(runner=contract_runner, paths=paths, sleeper=time.sleep)
|
||||||
|
executed = True
|
||||||
|
systemctl("start", contract_runner)
|
||||||
|
final = stable_service_posture(
|
||||||
|
release,
|
||||||
|
runner=contract_runner,
|
||||||
|
paths=paths,
|
||||||
|
sleeper=time.sleep,
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
final.kind == "active"
|
||||||
|
and final.state.get("NRestarts") == checked["prior_n_restarts"],
|
||||||
|
"prior active posture was not restored",
|
||||||
|
)
|
||||||
|
verification = verify_running_release(
|
||||||
|
release,
|
||||||
|
runner=contract_runner,
|
||||||
|
paths=paths,
|
||||||
|
sleeper=time.sleep,
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
verification.get("main_pid") == final.state.get("MainPID")
|
||||||
|
and verification.get("invocation_id") == final.state.get("InvocationID")
|
||||||
|
and verification.get("n_restarts") == checked["prior_n_restarts"],
|
||||||
|
"restored active service identity was unstable",
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise PostureRecoveryFailure(executed=executed) from exc
|
||||||
|
return _rollback_summary(
|
||||||
|
status="pass",
|
||||||
|
mode="posture_recovery",
|
||||||
|
executed=executed,
|
||||||
|
restored_posture=desired,
|
||||||
|
prior_posture_preserved=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _operational_failure_result(
|
def _operational_failure_result(
|
||||||
*,
|
*,
|
||||||
manifest: Mapping[str, Any],
|
manifest: Mapping[str, Any],
|
||||||
release: Mapping[str, Any],
|
release: Mapping[str, Any],
|
||||||
failure_stage: str,
|
failure_stage: str,
|
||||||
failure_reason: str,
|
failure_reason: str,
|
||||||
rollback: Mapping[str, str],
|
rollback: Mapping[str, Any],
|
||||||
fail_closed: Mapping[str, Any],
|
fail_closed: Mapping[str, Any],
|
||||||
archive_sha256: str,
|
archive_sha256: str,
|
||||||
helper_sha256: str,
|
helper_sha256: str,
|
||||||
|
|
@ -1006,7 +1323,8 @@ def _operational_failure_result(
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
_require(
|
_require(
|
||||||
manifest.get("operation") == "install"
|
manifest.get("operation") == "install"
|
||||||
and set(rollback) == {"status", "mode", "restored_posture"}
|
and set(rollback)
|
||||||
|
== {"status", "mode", "executed", "restored_posture", "prior_posture_preserved"}
|
||||||
and (
|
and (
|
||||||
(failure_stage, failure_reason) in POST_INSTALL_FAILURES
|
(failure_stage, failure_reason) in POST_INSTALL_FAILURES
|
||||||
or (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES
|
or (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES
|
||||||
|
|
@ -1015,14 +1333,22 @@ def _operational_failure_result(
|
||||||
)
|
)
|
||||||
rollback_passed = (
|
rollback_passed = (
|
||||||
rollback.get("status") == "pass"
|
rollback.get("status") == "pass"
|
||||||
and rollback.get("mode") == "explicit"
|
and rollback.get("mode") in {"explicit", "posture_recovery"}
|
||||||
|
and isinstance(rollback.get("executed"), bool)
|
||||||
and rollback.get("restored_posture") in {"active", "inactive", "absent"}
|
and rollback.get("restored_posture") in {"active", "inactive", "absent"}
|
||||||
|
and (
|
||||||
|
rollback.get("mode") == "explicit"
|
||||||
|
or rollback.get("restored_posture") in {"active", "inactive"}
|
||||||
|
)
|
||||||
|
and rollback.get("prior_posture_preserved") is True
|
||||||
|
)
|
||||||
|
rollback_failed = (
|
||||||
|
rollback.get("status") == "fail"
|
||||||
|
and rollback.get("mode") in {"explicit", "posture_recovery"}
|
||||||
|
and isinstance(rollback.get("executed"), bool)
|
||||||
|
and rollback.get("restored_posture") == "unknown"
|
||||||
|
and rollback.get("prior_posture_preserved") is False
|
||||||
)
|
)
|
||||||
rollback_failed = rollback == {
|
|
||||||
"status": "fail",
|
|
||||||
"mode": "explicit",
|
|
||||||
"restored_posture": "unknown",
|
|
||||||
}
|
|
||||||
verification_pair = (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES
|
verification_pair = (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES
|
||||||
_require(
|
_require(
|
||||||
(rollback_passed and not verification_pair) or rollback_failed,
|
(rollback_passed and not verification_pair) or rollback_failed,
|
||||||
|
|
@ -1045,6 +1371,7 @@ def _operational_failure_result(
|
||||||
"main_pid": None,
|
"main_pid": None,
|
||||||
"n_restarts": None,
|
"n_restarts": None,
|
||||||
"candidate_container": "unknown",
|
"candidate_container": "unknown",
|
||||||
|
"service_action_executed": fail_closed.get("service_action_executed"),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"status": "pass",
|
"status": "pass",
|
||||||
|
|
@ -1054,6 +1381,7 @@ def _operational_failure_result(
|
||||||
"main_pid": 0,
|
"main_pid": 0,
|
||||||
"n_restarts": fail_closed.get("n_restarts"),
|
"n_restarts": fail_closed.get("n_restarts"),
|
||||||
"candidate_container": "absent",
|
"candidate_container": "absent",
|
||||||
|
"service_action_executed": fail_closed.get("service_action_executed"),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
and (
|
and (
|
||||||
|
|
@ -1063,7 +1391,8 @@ def _operational_failure_result(
|
||||||
and not isinstance(fail_closed.get("n_restarts"), bool)
|
and not isinstance(fail_closed.get("n_restarts"), bool)
|
||||||
and fail_closed["n_restarts"] >= 0
|
and fail_closed["n_restarts"] >= 0
|
||||||
)
|
)
|
||||||
),
|
)
|
||||||
|
and isinstance(fail_closed.get("service_action_executed"), bool),
|
||||||
"post-install fail-closed summary was invalid",
|
"post-install fail-closed summary was invalid",
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
|
|
@ -1156,7 +1485,7 @@ def execute_request(
|
||||||
manifest=manifest,
|
manifest=manifest,
|
||||||
release=release,
|
release=release,
|
||||||
failure=exc,
|
failure=exc,
|
||||||
rollback={"status": "pass", "mode": "installer_automatic", "restored_posture": "prior"},
|
rollback=_installer_failure_rollback(exc),
|
||||||
archive_sha256=archive_sha256,
|
archive_sha256=archive_sha256,
|
||||||
helper_sha256=helper_sha256,
|
helper_sha256=helper_sha256,
|
||||||
installer_module=installer_module,
|
installer_module=installer_module,
|
||||||
|
|
@ -1189,8 +1518,13 @@ def execute_request(
|
||||||
if operation == "install":
|
if operation == "install":
|
||||||
failure_stage = "install_receipt"
|
failure_stage = "install_receipt"
|
||||||
failure_reason = "install_receipt_invalid"
|
failure_reason = "install_receipt_invalid"
|
||||||
|
installer_transition: dict[str, Any] | None = None
|
||||||
try:
|
try:
|
||||||
_require(install_receipt.get("status") == "pass", "installation did not pass")
|
installer_transition = _validate_install_execution_result(
|
||||||
|
install_receipt,
|
||||||
|
release=release,
|
||||||
|
installer_module=installer_module,
|
||||||
|
)
|
||||||
failure_stage = "service_before_restart"
|
failure_stage = "service_before_restart"
|
||||||
failure_reason = "service_verification_failed"
|
failure_reason = "service_verification_failed"
|
||||||
before = installer_module.verify_running_release(
|
before = installer_module.verify_running_release(
|
||||||
|
|
@ -1248,21 +1582,76 @@ def execute_request(
|
||||||
fallback_stage=failure_stage,
|
fallback_stage=failure_stage,
|
||||||
fallback_reason=failure_reason,
|
fallback_reason=failure_reason,
|
||||||
)
|
)
|
||||||
try:
|
if installer_transition is None:
|
||||||
recovery = _execute_explicit_rollback(
|
fail_closed = _fail_closed_inert_service(
|
||||||
installer_module=installer_module,
|
installer_module=installer_module,
|
||||||
release_sha256=release["release_sha256"],
|
release=release,
|
||||||
paths=paths,
|
paths=paths,
|
||||||
source_binding=source_binding,
|
|
||||||
runner=runner,
|
runner=runner,
|
||||||
)
|
)
|
||||||
rollback_result = _rollback_result(
|
raise RemoteOperationFailure(
|
||||||
manifest=manifest,
|
_operational_failure_result(
|
||||||
release=release,
|
manifest=manifest,
|
||||||
rollback=recovery,
|
release=release,
|
||||||
archive_sha256=archive_sha256,
|
failure_stage="install_receipt",
|
||||||
helper_sha256=helper_sha256,
|
failure_reason="install_receipt_invalid",
|
||||||
)
|
rollback=_rollback_summary(
|
||||||
|
status="fail",
|
||||||
|
mode="posture_recovery",
|
||||||
|
executed=bool(fail_closed["service_action_executed"]),
|
||||||
|
restored_posture="unknown",
|
||||||
|
prior_posture_preserved=False,
|
||||||
|
),
|
||||||
|
fail_closed=fail_closed,
|
||||||
|
archive_sha256=archive_sha256,
|
||||||
|
helper_sha256=helper_sha256,
|
||||||
|
installer_module=installer_module,
|
||||||
|
)
|
||||||
|
) from exc
|
||||||
|
release_changing = installer_transition["outcome"] in {"installed", "replaced"}
|
||||||
|
recovery_mode = "explicit" if release_changing else "posture_recovery"
|
||||||
|
recovery_executed = False
|
||||||
|
try:
|
||||||
|
if release_changing:
|
||||||
|
try:
|
||||||
|
recovery = _execute_explicit_rollback(
|
||||||
|
installer_module=installer_module,
|
||||||
|
release_sha256=release["release_sha256"],
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding,
|
||||||
|
runner=runner,
|
||||||
|
)
|
||||||
|
except ExplicitRollbackFailure as rollback_exc:
|
||||||
|
recovery_executed = rollback_exc.executed
|
||||||
|
raise
|
||||||
|
recovery_executed = True
|
||||||
|
rollback_result = _rollback_result(
|
||||||
|
manifest=manifest,
|
||||||
|
release=release,
|
||||||
|
rollback=recovery,
|
||||||
|
archive_sha256=archive_sha256,
|
||||||
|
helper_sha256=helper_sha256,
|
||||||
|
)
|
||||||
|
rollback_summary = _rollback_summary(
|
||||||
|
status="pass",
|
||||||
|
mode="explicit",
|
||||||
|
executed=True,
|
||||||
|
restored_posture=str(rollback_result["service"]["restored_posture"]),
|
||||||
|
prior_posture_preserved=True,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
rollback_summary = _restore_transition_posture(
|
||||||
|
installer_module=installer_module,
|
||||||
|
release=release,
|
||||||
|
transition=installer_transition,
|
||||||
|
paths=paths,
|
||||||
|
runner=runner,
|
||||||
|
)
|
||||||
|
except PostureRecoveryFailure as posture_exc:
|
||||||
|
recovery_executed = posture_exc.executed
|
||||||
|
raise
|
||||||
|
recovery_executed = bool(rollback_summary["executed"])
|
||||||
except Exception:
|
except Exception:
|
||||||
fail_closed = _fail_closed_inert_service(
|
fail_closed = _fail_closed_inert_service(
|
||||||
installer_module=installer_module,
|
installer_module=installer_module,
|
||||||
|
|
@ -1276,11 +1665,13 @@ def execute_request(
|
||||||
release=release,
|
release=release,
|
||||||
failure_stage=bounded_stage,
|
failure_stage=bounded_stage,
|
||||||
failure_reason=bounded_reason,
|
failure_reason=bounded_reason,
|
||||||
rollback={
|
rollback=_rollback_summary(
|
||||||
"status": "fail",
|
status="fail",
|
||||||
"mode": "explicit",
|
mode=recovery_mode,
|
||||||
"restored_posture": "unknown",
|
executed=recovery_executed,
|
||||||
},
|
restored_posture="unknown",
|
||||||
|
prior_posture_preserved=False,
|
||||||
|
),
|
||||||
fail_closed=fail_closed,
|
fail_closed=fail_closed,
|
||||||
archive_sha256=archive_sha256,
|
archive_sha256=archive_sha256,
|
||||||
helper_sha256=helper_sha256,
|
helper_sha256=helper_sha256,
|
||||||
|
|
@ -1293,28 +1684,20 @@ def execute_request(
|
||||||
manifest=manifest,
|
manifest=manifest,
|
||||||
release=release,
|
release=release,
|
||||||
failure=exc,
|
failure=exc,
|
||||||
rollback={
|
rollback=rollback_summary,
|
||||||
"status": "pass",
|
|
||||||
"mode": "explicit",
|
|
||||||
"restored_posture": rollback_result["service"]["restored_posture"],
|
|
||||||
},
|
|
||||||
archive_sha256=archive_sha256,
|
archive_sha256=archive_sha256,
|
||||||
helper_sha256=helper_sha256,
|
helper_sha256=helper_sha256,
|
||||||
installer_module=installer_module,
|
installer_module=installer_module,
|
||||||
)
|
)
|
||||||
) from exc
|
) from exc
|
||||||
restored_posture = str(rollback_result["service"]["restored_posture"])
|
restored_posture = str(rollback_summary["restored_posture"])
|
||||||
raise RemoteOperationFailure(
|
raise RemoteOperationFailure(
|
||||||
_operational_failure_result(
|
_operational_failure_result(
|
||||||
manifest=manifest,
|
manifest=manifest,
|
||||||
release=release,
|
release=release,
|
||||||
failure_stage=bounded_stage,
|
failure_stage=bounded_stage,
|
||||||
failure_reason=bounded_reason,
|
failure_reason=bounded_reason,
|
||||||
rollback={
|
rollback=rollback_summary,
|
||||||
"status": "pass",
|
|
||||||
"mode": "explicit",
|
|
||||||
"restored_posture": restored_posture,
|
|
||||||
},
|
|
||||||
fail_closed=_rollback_proven_service(restored_posture),
|
fail_closed=_rollback_proven_service(restored_posture),
|
||||||
archive_sha256=archive_sha256,
|
archive_sha256=archive_sha256,
|
||||||
helper_sha256=helper_sha256,
|
helper_sha256=helper_sha256,
|
||||||
|
|
@ -1323,6 +1706,7 @@ def execute_request(
|
||||||
) from exc
|
) from exc
|
||||||
service = {
|
service = {
|
||||||
"name": SERVICE,
|
"name": SERVICE,
|
||||||
|
"installer_transition": installer_transition,
|
||||||
"before_restart": before,
|
"before_restart": before,
|
||||||
"after_restart": after,
|
"after_restart": after,
|
||||||
}
|
}
|
||||||
|
|
@ -1346,7 +1730,13 @@ def execute_request(
|
||||||
manifest=manifest,
|
manifest=manifest,
|
||||||
release=release,
|
release=release,
|
||||||
failure=exc,
|
failure=exc,
|
||||||
rollback={"status": "not_run", "mode": "not_applicable", "restored_posture": "unchanged"},
|
rollback=_rollback_summary(
|
||||||
|
status="not_run",
|
||||||
|
mode="not_applicable",
|
||||||
|
executed=False,
|
||||||
|
restored_posture="unchanged",
|
||||||
|
prior_posture_preserved=True,
|
||||||
|
),
|
||||||
archive_sha256=archive_sha256,
|
archive_sha256=archive_sha256,
|
||||||
helper_sha256=helper_sha256,
|
helper_sha256=helper_sha256,
|
||||||
installer_module=installer_module,
|
installer_module=installer_module,
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,9 @@ if str(CONTRACT_ROOT) not in sys.path:
|
||||||
from ops import gcp_leoclean_nosend_package as package # noqa: E402
|
from ops import gcp_leoclean_nosend_package as package # noqa: E402
|
||||||
from ops import leoclean_nosend_service_control as service_control # noqa: E402
|
from ops import leoclean_nosend_service_control as service_control # noqa: E402
|
||||||
|
|
||||||
INSTALL_RECEIPT_SCHEMA = "livingip.leocleanNoSendInstallReceipt.v2"
|
LEGACY_INSTALL_RECEIPT_SCHEMA = "livingip.leocleanNoSendInstallReceipt.v2"
|
||||||
|
INSTALL_RECEIPT_SCHEMA = "livingip.leocleanNoSendInstallReceipt.v3"
|
||||||
|
INSTALL_EXECUTION_RESULT_SCHEMA = "livingip.leocleanNoSendInstallExecution.v1"
|
||||||
ROLLBACK_STATE_SCHEMA = "livingip.leocleanNoSendRollbackState.v1"
|
ROLLBACK_STATE_SCHEMA = "livingip.leocleanNoSendRollbackState.v1"
|
||||||
ROLLBACK_RESULT_SCHEMA = "livingip.leocleanNoSendRollbackResult.v1"
|
ROLLBACK_RESULT_SCHEMA = "livingip.leocleanNoSendRollbackResult.v1"
|
||||||
SERVICE = package.SERVICE
|
SERVICE = package.SERVICE
|
||||||
|
|
@ -171,12 +173,19 @@ class VerificationError(InstallError):
|
||||||
failure_stage: str,
|
failure_stage: str,
|
||||||
failure_reason: str,
|
failure_reason: str,
|
||||||
message: str = "release verification failed",
|
message: str = "release verification failed",
|
||||||
|
*,
|
||||||
|
rollback_executed: bool | None = None,
|
||||||
|
restored_posture: str | None = None,
|
||||||
|
prior_posture_preserved: bool | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
if (failure_stage, failure_reason) not in VERIFICATION_FAILURES:
|
if (failure_stage, failure_reason) not in VERIFICATION_FAILURES:
|
||||||
raise ValueError("verification failure code was not allowlisted")
|
raise ValueError("verification failure code was not allowlisted")
|
||||||
super().__init__(message)
|
super().__init__(message)
|
||||||
self.failure_stage = failure_stage
|
self.failure_stage = failure_stage
|
||||||
self.failure_reason = failure_reason
|
self.failure_reason = failure_reason
|
||||||
|
self.rollback_executed = rollback_executed
|
||||||
|
self.restored_posture = restored_posture
|
||||||
|
self.prior_posture_preserved = prior_posture_preserved
|
||||||
|
|
||||||
|
|
||||||
class AtomicWriteError(InstallError):
|
class AtomicWriteError(InstallError):
|
||||||
|
|
@ -284,6 +293,93 @@ class ServicePosture:
|
||||||
state: dict[str, Any]
|
state: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
def _install_transition(
|
||||||
|
outcome: str,
|
||||||
|
*,
|
||||||
|
prior_posture: str,
|
||||||
|
prior_n_restarts: int,
|
||||||
|
service_action: str,
|
||||||
|
managed_files_replaced: bool,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
_require(
|
||||||
|
outcome in {"verified_noop", "controlled_start", "installed", "replaced"}
|
||||||
|
and prior_posture in {"absent", "inactive", "active"}
|
||||||
|
and isinstance(prior_n_restarts, int)
|
||||||
|
and not isinstance(prior_n_restarts, bool)
|
||||||
|
and prior_n_restarts >= 0
|
||||||
|
and service_action in {"none", "start", "restart"}
|
||||||
|
and (
|
||||||
|
(outcome == "verified_noop" and prior_posture == "active" and service_action == "none")
|
||||||
|
or (outcome == "controlled_start" and prior_posture == "inactive" and service_action == "start")
|
||||||
|
or (outcome == "installed" and prior_posture == "absent" and service_action == "restart")
|
||||||
|
or (outcome == "replaced" and prior_posture in {"inactive", "active"} and service_action == "restart")
|
||||||
|
)
|
||||||
|
and managed_files_replaced == (outcome in {"installed", "replaced"}),
|
||||||
|
"installer transition was invalid",
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"outcome": outcome,
|
||||||
|
"prior_posture": prior_posture,
|
||||||
|
"prior_n_restarts": prior_n_restarts,
|
||||||
|
"final_posture": "active",
|
||||||
|
"service_action": service_action,
|
||||||
|
"managed_files_replaced": managed_files_replaced,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _install_execution_result(
|
||||||
|
*,
|
||||||
|
outcome: str,
|
||||||
|
release: Mapping[str, Any],
|
||||||
|
receipt: Mapping[str, Any],
|
||||||
|
receipt_raw: bytes,
|
||||||
|
transition: Mapping[str, Any],
|
||||||
|
verification: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
_require(
|
||||||
|
outcome in {"verified_noop", "controlled_start", "installed", "replaced"}
|
||||||
|
and receipt.get("status") == "pass"
|
||||||
|
and receipt.get("release_sha256") == release["release_sha256"]
|
||||||
|
and (
|
||||||
|
(
|
||||||
|
outcome in {"verified_noop", "controlled_start"}
|
||||||
|
and receipt.get("schema") in {LEGACY_INSTALL_RECEIPT_SCHEMA, INSTALL_RECEIPT_SCHEMA}
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
outcome in {"installed", "replaced"}
|
||||||
|
and receipt.get("schema") == INSTALL_RECEIPT_SCHEMA
|
||||||
|
and receipt.get("outcome") == outcome
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"exact-release execution result was invalid",
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema": INSTALL_EXECUTION_RESULT_SCHEMA,
|
||||||
|
"status": "pass",
|
||||||
|
"outcome": outcome,
|
||||||
|
"release_sha256": release["release_sha256"],
|
||||||
|
"persisted_receipt": {
|
||||||
|
"schema": receipt["schema"],
|
||||||
|
"sha256": _sha256_bytes(receipt_raw),
|
||||||
|
},
|
||||||
|
"transition": dict(transition),
|
||||||
|
"verification": dict(verification),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _attach_recovery(
|
||||||
|
failure: Exception,
|
||||||
|
*,
|
||||||
|
rollback_executed: bool,
|
||||||
|
restored_posture: str,
|
||||||
|
prior_posture_preserved: bool,
|
||||||
|
) -> None:
|
||||||
|
if isinstance(failure, VerificationError):
|
||||||
|
failure.rollback_executed = rollback_executed
|
||||||
|
failure.restored_posture = restored_posture
|
||||||
|
failure.prior_posture_preserved = prior_posture_preserved
|
||||||
|
|
||||||
|
|
||||||
def _sha256_bytes(value: bytes) -> str:
|
def _sha256_bytes(value: bytes) -> str:
|
||||||
return hashlib.sha256(value).hexdigest()
|
return hashlib.sha256(value).hexdigest()
|
||||||
|
|
||||||
|
|
@ -1558,17 +1654,228 @@ def _read_rollback_state(paths: InstallPaths, release_sha256: str) -> tuple[dict
|
||||||
return value, raw
|
return value, raw
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_current_install_receipt_contract(
|
||||||
|
receipt: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
release: Mapping[str, Any],
|
||||||
|
rollback_state: Mapping[str, Any],
|
||||||
|
) -> None:
|
||||||
|
_require(
|
||||||
|
set(receipt)
|
||||||
|
== {
|
||||||
|
"schema",
|
||||||
|
"status",
|
||||||
|
"outcome",
|
||||||
|
"transition",
|
||||||
|
"installer_source",
|
||||||
|
"release_sha256",
|
||||||
|
"image",
|
||||||
|
"installed_files",
|
||||||
|
"service_before",
|
||||||
|
"verification",
|
||||||
|
"rollback",
|
||||||
|
"authority",
|
||||||
|
},
|
||||||
|
"installed receipt fields were not exact",
|
||||||
|
)
|
||||||
|
rollback = receipt.get("rollback")
|
||||||
|
_require(
|
||||||
|
isinstance(rollback, dict)
|
||||||
|
and set(rollback)
|
||||||
|
== {
|
||||||
|
"previous_release_present",
|
||||||
|
"previous_service_active",
|
||||||
|
"previous_service_posture",
|
||||||
|
"automatic_on_failure",
|
||||||
|
"state_path",
|
||||||
|
"state_sha256",
|
||||||
|
}
|
||||||
|
and isinstance(rollback.get("previous_release_present"), bool)
|
||||||
|
and isinstance(rollback.get("previous_service_active"), bool)
|
||||||
|
and isinstance(rollback.get("automatic_on_failure"), bool)
|
||||||
|
and rollback["automatic_on_failure"] is True,
|
||||||
|
"installed receipt rollback fields were invalid",
|
||||||
|
)
|
||||||
|
previous_files = rollback_state["previous_files"]
|
||||||
|
previous_service = rollback_state["previous_service"]
|
||||||
|
previous_release_present = all(
|
||||||
|
_snapshot_from_json(previous_files[role], label=f"rollback {role} snapshot").existed
|
||||||
|
for role in sorted(previous_files)
|
||||||
|
)
|
||||||
|
prior_posture = previous_service["posture"]
|
||||||
|
prior_n_restarts = previous_service["n_restarts"]
|
||||||
|
outcome = receipt.get("outcome")
|
||||||
|
_require(
|
||||||
|
(
|
||||||
|
outcome == "installed"
|
||||||
|
and prior_posture == "absent"
|
||||||
|
and previous_release_present is False
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
outcome == "replaced"
|
||||||
|
and prior_posture in {"active", "inactive"}
|
||||||
|
and previous_release_present is True
|
||||||
|
),
|
||||||
|
"installed receipt outcome differed from rollback state",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
rollback["previous_release_present"] is previous_release_present
|
||||||
|
and rollback["previous_service_active"] is (prior_posture == "active")
|
||||||
|
and rollback["previous_service_posture"] == prior_posture,
|
||||||
|
"installed receipt prior posture differed from rollback state",
|
||||||
|
)
|
||||||
|
service_before = receipt.get("service_before")
|
||||||
|
_require(
|
||||||
|
isinstance(service_before, dict)
|
||||||
|
and set(service_before)
|
||||||
|
== {
|
||||||
|
"load_state",
|
||||||
|
"active_state",
|
||||||
|
"sub_state",
|
||||||
|
"main_pid",
|
||||||
|
"n_restarts",
|
||||||
|
}
|
||||||
|
and isinstance(service_before.get("main_pid"), int)
|
||||||
|
and not isinstance(service_before.get("main_pid"), bool)
|
||||||
|
and service_before["main_pid"] >= 0
|
||||||
|
and isinstance(service_before.get("n_restarts"), int)
|
||||||
|
and not isinstance(service_before.get("n_restarts"), bool)
|
||||||
|
and service_before["n_restarts"] >= 0
|
||||||
|
and service_before["active_state"] == previous_service["active_state"]
|
||||||
|
and service_before["sub_state"] == previous_service["sub_state"]
|
||||||
|
and service_before["n_restarts"] == prior_n_restarts,
|
||||||
|
"installed receipt service posture differed from rollback state",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
(
|
||||||
|
prior_posture == "absent"
|
||||||
|
and service_before["load_state"] == "not-found"
|
||||||
|
and service_before["main_pid"] == 0
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
prior_posture == "inactive"
|
||||||
|
and service_before["load_state"] == "loaded"
|
||||||
|
and service_before["main_pid"] == 0
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
prior_posture == "active"
|
||||||
|
and service_before["load_state"] == "loaded"
|
||||||
|
and service_before["main_pid"] > 1
|
||||||
|
),
|
||||||
|
"installed receipt load or process posture differed from rollback state",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
receipt.get("transition")
|
||||||
|
== _install_transition(
|
||||||
|
str(outcome),
|
||||||
|
prior_posture=prior_posture,
|
||||||
|
prior_n_restarts=prior_n_restarts,
|
||||||
|
service_action="restart",
|
||||||
|
managed_files_replaced=True,
|
||||||
|
),
|
||||||
|
"installed receipt transition was invalid",
|
||||||
|
)
|
||||||
|
verification = receipt.get("verification")
|
||||||
|
_require(
|
||||||
|
isinstance(verification, dict)
|
||||||
|
and set(verification)
|
||||||
|
== {
|
||||||
|
"active_state",
|
||||||
|
"sub_state",
|
||||||
|
"main_pid",
|
||||||
|
"n_restarts",
|
||||||
|
"invocation_id",
|
||||||
|
"supervising_process",
|
||||||
|
"container_id",
|
||||||
|
"container_pid",
|
||||||
|
"health",
|
||||||
|
"validated_process_environment_fields",
|
||||||
|
"dropins",
|
||||||
|
"environment_files",
|
||||||
|
}
|
||||||
|
and verification.get("active_state") == "active"
|
||||||
|
and verification.get("sub_state") == "running"
|
||||||
|
and isinstance(verification.get("main_pid"), int)
|
||||||
|
and not isinstance(verification.get("main_pid"), bool)
|
||||||
|
and verification["main_pid"] > 1
|
||||||
|
and isinstance(verification.get("n_restarts"), int)
|
||||||
|
and not isinstance(verification.get("n_restarts"), bool)
|
||||||
|
and verification["n_restarts"] >= 0
|
||||||
|
and isinstance(verification.get("invocation_id"), str)
|
||||||
|
and HEX_32.fullmatch(verification["invocation_id"]) is not None
|
||||||
|
and isinstance(verification.get("container_id"), str)
|
||||||
|
and HEX_64.fullmatch(verification["container_id"]) is not None
|
||||||
|
and isinstance(verification.get("container_pid"), int)
|
||||||
|
and not isinstance(verification.get("container_pid"), bool)
|
||||||
|
and verification["container_pid"] > 1
|
||||||
|
and verification.get("health") == "healthy"
|
||||||
|
and verification.get("dropins") == "absent"
|
||||||
|
and verification.get("environment_files") == "absent",
|
||||||
|
"installed receipt verification was invalid",
|
||||||
|
)
|
||||||
|
supervising_process = verification["supervising_process"]
|
||||||
|
environment_fields = verification["validated_process_environment_fields"]
|
||||||
|
expected_start = shlex.split(_rendered_service_fields(release)["ExecStart"][0])
|
||||||
|
expected_executable = expected_start[0]
|
||||||
|
expected_cmdline = b"\0".join(item.encode("utf-8") for item in expected_start) + b"\0"
|
||||||
|
_require(
|
||||||
|
isinstance(supervising_process, dict)
|
||||||
|
and set(supervising_process) == {"executable", "cmdline_sha256", "cgroup"}
|
||||||
|
and isinstance(supervising_process.get("executable"), str)
|
||||||
|
and supervising_process["executable"] in {expected_executable, str(Path(expected_executable).resolve())}
|
||||||
|
and isinstance(supervising_process.get("cmdline_sha256"), str)
|
||||||
|
and supervising_process["cmdline_sha256"] == _sha256_bytes(expected_cmdline)
|
||||||
|
and supervising_process.get("cgroup") == f"/system.slice/{SERVICE}"
|
||||||
|
and isinstance(environment_fields, list)
|
||||||
|
and all(isinstance(field, str) and field for field in environment_fields)
|
||||||
|
and environment_fields == sorted(set(environment_fields))
|
||||||
|
and "PATH" in environment_fields,
|
||||||
|
"installed receipt verification detail was invalid",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
receipt.get("authority")
|
||||||
|
== {
|
||||||
|
"image_pulled": False,
|
||||||
|
"cloud_or_database_authority_changed": False,
|
||||||
|
"transport_authority_changed": False,
|
||||||
|
},
|
||||||
|
"installed receipt authority was invalid",
|
||||||
|
)
|
||||||
|
installed_files = receipt.get("installed_files")
|
||||||
|
_require(
|
||||||
|
isinstance(installed_files, dict)
|
||||||
|
and set(installed_files)
|
||||||
|
== {
|
||||||
|
"unit_sha256",
|
||||||
|
"helper_sha256",
|
||||||
|
"release_descriptor_sha256",
|
||||||
|
"rollback_state_sha256",
|
||||||
|
},
|
||||||
|
"installed receipt file fields were not exact",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _validate_install_receipt(
|
def _validate_install_receipt(
|
||||||
receipt: Mapping[str, Any],
|
receipt: Mapping[str, Any],
|
||||||
*,
|
*,
|
||||||
release: Mapping[str, Any],
|
release: Mapping[str, Any],
|
||||||
paths: InstallPaths,
|
paths: InstallPaths,
|
||||||
|
rollback_state: Mapping[str, Any],
|
||||||
rollback_state_raw: bytes,
|
rollback_state_raw: bytes,
|
||||||
) -> None:
|
) -> None:
|
||||||
installed_files = receipt.get("installed_files")
|
installed_files = receipt.get("installed_files")
|
||||||
rollback = receipt.get("rollback")
|
rollback = receipt.get("rollback")
|
||||||
|
schema = receipt.get("schema")
|
||||||
|
legacy_result_valid = schema == LEGACY_INSTALL_RECEIPT_SCHEMA and receipt.get("result") == "installed"
|
||||||
|
if schema == INSTALL_RECEIPT_SCHEMA:
|
||||||
|
_validate_current_install_receipt_contract(
|
||||||
|
receipt,
|
||||||
|
release=release,
|
||||||
|
rollback_state=rollback_state,
|
||||||
|
)
|
||||||
|
current_outcome_valid = schema == INSTALL_RECEIPT_SCHEMA
|
||||||
_require(
|
_require(
|
||||||
receipt.get("schema") == INSTALL_RECEIPT_SCHEMA
|
(legacy_result_valid or current_outcome_valid)
|
||||||
and receipt.get("status") == "pass"
|
and receipt.get("status") == "pass"
|
||||||
and receipt.get("release_sha256") == release["release_sha256"]
|
and receipt.get("release_sha256") == release["release_sha256"]
|
||||||
and isinstance(installed_files, dict)
|
and isinstance(installed_files, dict)
|
||||||
|
|
@ -1644,7 +1951,6 @@ def _validate_managed_prior_state(paths: InstallPaths, helper_source: bytes) ->
|
||||||
_require(paths.unit.read_bytes() == package.render_systemd_unit(release).encode(), "installed unit drifted")
|
_require(paths.unit.read_bytes() == package.render_systemd_unit(release).encode(), "installed unit drifted")
|
||||||
_require(paths.helper.is_file() and not paths.helper.is_symlink(), "installed helper was unsafe")
|
_require(paths.helper.is_file() and not paths.helper.is_symlink(), "installed helper was unsafe")
|
||||||
rollback_state, rollback_state_raw = _read_rollback_state(paths, release["release_sha256"])
|
rollback_state, rollback_state_raw = _read_rollback_state(paths, release["release_sha256"])
|
||||||
del rollback_state
|
|
||||||
receipt, _receipt_raw = _read_strict_json_file(
|
receipt, _receipt_raw = _read_strict_json_file(
|
||||||
paths.receipt,
|
paths.receipt,
|
||||||
label="install receipt",
|
label="install receipt",
|
||||||
|
|
@ -1655,6 +1961,7 @@ def _validate_managed_prior_state(paths: InstallPaths, helper_source: bytes) ->
|
||||||
receipt,
|
receipt,
|
||||||
release=release,
|
release=release,
|
||||||
paths=paths,
|
paths=paths,
|
||||||
|
rollback_state=rollback_state,
|
||||||
rollback_state_raw=rollback_state_raw,
|
rollback_state_raw=rollback_state_raw,
|
||||||
)
|
)
|
||||||
_require(bool(helper_source), "service-control helper source was empty")
|
_require(bool(helper_source), "service-control helper source was empty")
|
||||||
|
|
@ -1665,7 +1972,7 @@ def _systemctl(action: str, runner: Runner) -> None:
|
||||||
if action == "daemon-reload":
|
if action == "daemon-reload":
|
||||||
argv = [SYSTEMCTL, "daemon-reload"]
|
argv = [SYSTEMCTL, "daemon-reload"]
|
||||||
else:
|
else:
|
||||||
_require(action in {"restart", "stop", "reset-failed"}, "systemd action was not allowed")
|
_require(action in {"start", "restart", "stop", "reset-failed"}, "systemd action was not allowed")
|
||||||
argv = [SYSTEMCTL, action, SERVICE]
|
argv = [SYSTEMCTL, action, SERVICE]
|
||||||
_run_checked(argv, runner, label=f"systemd {action}")
|
_run_checked(argv, runner, label=f"systemd {action}")
|
||||||
|
|
||||||
|
|
@ -1817,11 +2124,26 @@ def _install_release_transaction(
|
||||||
_ensure_directory(paths.state_dir, 0o700, created)
|
_ensure_directory(paths.state_dir, 0o700, created)
|
||||||
_ensure_directory(paths.docker_config, 0o700, created)
|
_ensure_directory(paths.docker_config, 0o700, created)
|
||||||
_require(not any(paths.docker_config.iterdir()), "service Docker config directory was not empty")
|
_require(not any(paths.docker_config.iterdir()), "service Docker config directory was not empty")
|
||||||
_validate_local_image(release, runner, paths)
|
try:
|
||||||
|
_validate_local_image(release, runner, paths)
|
||||||
|
except VerificationError:
|
||||||
|
raise
|
||||||
|
except InstallError as exc:
|
||||||
|
raise VerificationError(
|
||||||
|
"image_identity",
|
||||||
|
"image_contract_mismatch",
|
||||||
|
str(exc),
|
||||||
|
) from exc
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
cleanup_complete = _cleanup_created_directories(created)
|
cleanup_complete = _cleanup_created_directories(created)
|
||||||
if not cleanup_complete:
|
if not cleanup_complete:
|
||||||
raise InstallError("preflight failed and cleanup was incomplete") from exc
|
raise InstallError("preflight failed and cleanup was incomplete") from exc
|
||||||
|
_attach_recovery(
|
||||||
|
exc,
|
||||||
|
rollback_executed=False,
|
||||||
|
restored_posture=prior_posture.kind,
|
||||||
|
prior_posture_preserved=True,
|
||||||
|
)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
snapshots = {
|
snapshots = {
|
||||||
|
|
@ -1841,19 +2163,133 @@ def _install_release_transaction(
|
||||||
and snapshots[paths.helper].content == helper_content
|
and snapshots[paths.helper].content == helper_content
|
||||||
)
|
)
|
||||||
if exact_already_installed:
|
if exact_already_installed:
|
||||||
verified = verify_running_release(release, runner=runner, paths=paths, sleeper=sleeper)
|
exact_snapshots = _current_managed_snapshots(paths)
|
||||||
existing_receipt = package.load_json(paths.receipt, "install receipt")
|
exact_release, existing_receipt, _exact_rollback = _validate_managed_snapshots(
|
||||||
return {**existing_receipt, "result": "already_installed", "verification": verified}
|
exact_snapshots,
|
||||||
|
paths=paths,
|
||||||
|
)
|
||||||
|
_require(exact_release == release, "exact managed release changed before convergence")
|
||||||
|
receipt_raw = exact_snapshots["receipt"].content
|
||||||
|
if prior_posture.kind == "active":
|
||||||
|
try:
|
||||||
|
verified = verify_running_release(release, runner=runner, paths=paths, sleeper=sleeper)
|
||||||
|
_verification_require(
|
||||||
|
verified["main_pid"] == before["MainPID"]
|
||||||
|
and verified["n_restarts"] == before["NRestarts"]
|
||||||
|
and verified["invocation_id"] == before["InvocationID"],
|
||||||
|
"startup_identity",
|
||||||
|
"service_identity_changed",
|
||||||
|
"active exact release changed during verification",
|
||||||
|
)
|
||||||
|
_assert_snapshot_identities(exact_snapshots, paths=paths)
|
||||||
|
except Exception as exc:
|
||||||
|
_attach_recovery(
|
||||||
|
exc,
|
||||||
|
rollback_executed=False,
|
||||||
|
restored_posture="active",
|
||||||
|
prior_posture_preserved=True,
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
transition = _install_transition(
|
||||||
|
"verified_noop",
|
||||||
|
prior_posture="active",
|
||||||
|
prior_n_restarts=before["NRestarts"],
|
||||||
|
service_action="none",
|
||||||
|
managed_files_replaced=False,
|
||||||
|
)
|
||||||
|
return _install_execution_result(
|
||||||
|
outcome="verified_noop",
|
||||||
|
release=release,
|
||||||
|
receipt=existing_receipt,
|
||||||
|
receipt_raw=receipt_raw,
|
||||||
|
transition=transition,
|
||||||
|
verification=verified,
|
||||||
|
)
|
||||||
|
|
||||||
|
_require(prior_posture.kind == "inactive", "exact managed release posture was invalid")
|
||||||
|
start_attempted = False
|
||||||
|
try:
|
||||||
|
_require_candidate_absent(runner=runner, paths=paths, sleeper=sleeper)
|
||||||
|
start_attempted = True
|
||||||
|
try:
|
||||||
|
_systemctl("start", runner)
|
||||||
|
except InstallError as exc:
|
||||||
|
raise VerificationError(
|
||||||
|
"service_activation",
|
||||||
|
"service_not_active",
|
||||||
|
"controlled service start failed",
|
||||||
|
) from exc
|
||||||
|
verified = verify_running_release(release, runner=runner, paths=paths, sleeper=sleeper)
|
||||||
|
_verification_require(
|
||||||
|
verified["n_restarts"] == before["NRestarts"],
|
||||||
|
"startup_identity",
|
||||||
|
"service_identity_changed",
|
||||||
|
"controlled service start changed the restart counter",
|
||||||
|
)
|
||||||
|
_assert_snapshot_identities(exact_snapshots, paths=paths)
|
||||||
|
transition = _install_transition(
|
||||||
|
"controlled_start",
|
||||||
|
prior_posture="inactive",
|
||||||
|
prior_n_restarts=before["NRestarts"],
|
||||||
|
service_action="start",
|
||||||
|
managed_files_replaced=False,
|
||||||
|
)
|
||||||
|
return _install_execution_result(
|
||||||
|
outcome="controlled_start",
|
||||||
|
release=release,
|
||||||
|
receipt=existing_receipt,
|
||||||
|
receipt_raw=receipt_raw,
|
||||||
|
transition=transition,
|
||||||
|
verification=verified,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
try:
|
||||||
|
if start_attempted:
|
||||||
|
_systemctl("stop", runner)
|
||||||
|
_require_candidate_absent(runner=runner, paths=paths, sleeper=sleeper)
|
||||||
|
restored_posture = _stable_service_posture(
|
||||||
|
release,
|
||||||
|
runner=runner,
|
||||||
|
paths=paths,
|
||||||
|
sleeper=sleeper,
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
restored_posture.kind == "inactive"
|
||||||
|
and restored_posture.state["NRestarts"] == before["NRestarts"],
|
||||||
|
"controlled start rollback did not restore the prior inactive posture",
|
||||||
|
)
|
||||||
|
except InstallError as recovery_exc:
|
||||||
|
raise InstallError("controlled service start failed and cleanup was incomplete") from recovery_exc
|
||||||
|
_attach_recovery(
|
||||||
|
exc,
|
||||||
|
rollback_executed=start_attempted,
|
||||||
|
restored_posture="inactive",
|
||||||
|
prior_posture_preserved=True,
|
||||||
|
)
|
||||||
|
if isinstance(exc, InstallError):
|
||||||
|
raise
|
||||||
|
raise InstallError("controlled service start failed and prior posture was restored") from exc
|
||||||
|
|
||||||
if prior_active:
|
if prior_active:
|
||||||
_require(prior_release is not None, "active managed service had no prior release")
|
_require(prior_release is not None, "active managed service had no prior release")
|
||||||
prior_verification = verify_running_release(prior_release, runner=runner, paths=paths, sleeper=sleeper)
|
try:
|
||||||
_require(
|
prior_verification = verify_running_release(prior_release, runner=runner, paths=paths, sleeper=sleeper)
|
||||||
prior_verification["main_pid"] == before["MainPID"]
|
_verification_require(
|
||||||
and prior_verification["n_restarts"] == before["NRestarts"]
|
prior_verification["main_pid"] == before["MainPID"]
|
||||||
and prior_verification["invocation_id"] == before["InvocationID"],
|
and prior_verification["n_restarts"] == before["NRestarts"]
|
||||||
"active prior service changed before the trusted stop",
|
and prior_verification["invocation_id"] == before["InvocationID"],
|
||||||
)
|
"startup_identity",
|
||||||
|
"service_identity_changed",
|
||||||
|
"active prior service changed before the trusted stop",
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
_attach_recovery(
|
||||||
|
exc,
|
||||||
|
rollback_executed=False,
|
||||||
|
restored_posture="active",
|
||||||
|
prior_posture_preserved=True,
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
candidate_may_be_active = False
|
candidate_may_be_active = False
|
||||||
try:
|
try:
|
||||||
|
|
@ -1893,10 +2329,19 @@ def _install_release_transaction(
|
||||||
)
|
)
|
||||||
attempted_paths.add(paths.rollback_state)
|
attempted_paths.add(paths.rollback_state)
|
||||||
_publish_managed_file(paths.rollback_state, rollback_state_bytes, 0o600, installed_identities)
|
_publish_managed_file(paths.rollback_state, rollback_state_bytes, 0o600, installed_identities)
|
||||||
|
outcome = "installed" if prior_release is None else "replaced"
|
||||||
|
transition = _install_transition(
|
||||||
|
outcome,
|
||||||
|
prior_posture=prior_posture.kind,
|
||||||
|
prior_n_restarts=before["NRestarts"],
|
||||||
|
service_action="restart",
|
||||||
|
managed_files_replaced=True,
|
||||||
|
)
|
||||||
receipt = {
|
receipt = {
|
||||||
"schema": INSTALL_RECEIPT_SCHEMA,
|
"schema": INSTALL_RECEIPT_SCHEMA,
|
||||||
"status": "pass",
|
"status": "pass",
|
||||||
"result": "installed",
|
"outcome": outcome,
|
||||||
|
"transition": transition,
|
||||||
"installer_source": dict(source_binding),
|
"installer_source": dict(source_binding),
|
||||||
"release_sha256": release["release_sha256"],
|
"release_sha256": release["release_sha256"],
|
||||||
"image": {
|
"image": {
|
||||||
|
|
@ -1936,7 +2381,14 @@ def _install_release_transaction(
|
||||||
receipt_bytes = _canonical_json(receipt)
|
receipt_bytes = _canonical_json(receipt)
|
||||||
attempted_paths.add(paths.receipt)
|
attempted_paths.add(paths.receipt)
|
||||||
_publish_managed_file(paths.receipt, receipt_bytes, 0o600, installed_identities)
|
_publish_managed_file(paths.receipt, receipt_bytes, 0o600, installed_identities)
|
||||||
return receipt
|
return _install_execution_result(
|
||||||
|
outcome=outcome,
|
||||||
|
release=release,
|
||||||
|
receipt=receipt,
|
||||||
|
receipt_raw=receipt_bytes,
|
||||||
|
transition=transition,
|
||||||
|
verification=verified,
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
rollback_errors: list[str] = []
|
rollback_errors: list[str] = []
|
||||||
if candidate_may_be_active:
|
if candidate_may_be_active:
|
||||||
|
|
@ -1992,6 +2444,12 @@ def _install_release_transaction(
|
||||||
rollback_errors.append("directory_cleanup")
|
rollback_errors.append("directory_cleanup")
|
||||||
if rollback_errors:
|
if rollback_errors:
|
||||||
raise InstallError("installation failed and automatic rollback was incomplete") from exc
|
raise InstallError("installation failed and automatic rollback was incomplete") from exc
|
||||||
|
_attach_recovery(
|
||||||
|
exc,
|
||||||
|
rollback_executed=True,
|
||||||
|
restored_posture=prior_posture.kind,
|
||||||
|
prior_posture_preserved=True,
|
||||||
|
)
|
||||||
if isinstance(exc, InstallError):
|
if isinstance(exc, InstallError):
|
||||||
raise
|
raise
|
||||||
raise InstallError("installation failed and was rolled back") from exc
|
raise InstallError("installation failed and was rolled back") from exc
|
||||||
|
|
@ -2032,8 +2490,17 @@ def _validate_managed_snapshots(
|
||||||
receipt = _json_from_snapshot(snapshots["receipt"], label="snapshot install receipt")
|
receipt = _json_from_snapshot(snapshots["receipt"], label="snapshot install receipt")
|
||||||
installed_files = receipt.get("installed_files")
|
installed_files = receipt.get("installed_files")
|
||||||
rollback = receipt.get("rollback")
|
rollback = receipt.get("rollback")
|
||||||
|
schema = receipt.get("schema")
|
||||||
|
legacy_result_valid = schema == LEGACY_INSTALL_RECEIPT_SCHEMA and receipt.get("result") == "installed"
|
||||||
|
if schema == INSTALL_RECEIPT_SCHEMA:
|
||||||
|
_validate_current_install_receipt_contract(
|
||||||
|
receipt,
|
||||||
|
release=release,
|
||||||
|
rollback_state=rollback_state,
|
||||||
|
)
|
||||||
|
current_outcome_valid = schema == INSTALL_RECEIPT_SCHEMA
|
||||||
_require(
|
_require(
|
||||||
receipt.get("schema") == INSTALL_RECEIPT_SCHEMA
|
(legacy_result_valid or current_outcome_valid)
|
||||||
and receipt.get("status") == "pass"
|
and receipt.get("status") == "pass"
|
||||||
and receipt.get("release_sha256") == release_sha256
|
and receipt.get("release_sha256") == release_sha256
|
||||||
and isinstance(receipt.get("installer_source"), dict)
|
and isinstance(receipt.get("installer_source"), dict)
|
||||||
|
|
|
||||||
|
|
@ -31,9 +31,9 @@ from ops import install_gcp_leoclean_nosend_release as installer # noqa: E402
|
||||||
|
|
||||||
SCHEMA = "livingip.leocleanNoSendIapRequest.v1"
|
SCHEMA = "livingip.leocleanNoSendIapRequest.v1"
|
||||||
PLAN_SCHEMA = "livingip.leocleanNoSendIapPlan.v1"
|
PLAN_SCHEMA = "livingip.leocleanNoSendIapPlan.v1"
|
||||||
REMOTE_RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v1"
|
REMOTE_RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v2"
|
||||||
REMOTE_FAILURE_SCHEMA = "livingip.leocleanNoSendIapFailure.v1"
|
REMOTE_FAILURE_SCHEMA = "livingip.leocleanNoSendIapFailure.v2"
|
||||||
REMOTE_OPERATION_FAILURE_SCHEMA = "livingip.leocleanNoSendIapOperationalFailure.v1"
|
REMOTE_OPERATION_FAILURE_SCHEMA = "livingip.leocleanNoSendIapOperationalFailure.v2"
|
||||||
RESULT_RECEIPT_SCHEMA = "livingip.leocleanNoSendIapResultReceipt.v1"
|
RESULT_RECEIPT_SCHEMA = "livingip.leocleanNoSendIapResultReceipt.v1"
|
||||||
FAILURE_RECEIPT_SCHEMA = "livingip.leocleanNoSendIapFailureReceipt.v1"
|
FAILURE_RECEIPT_SCHEMA = "livingip.leocleanNoSendIapFailureReceipt.v1"
|
||||||
LOCAL_FAILURE_RECEIPT_SCHEMA = "livingip.leocleanNoSendIapLocalFailureReceipt.v1"
|
LOCAL_FAILURE_RECEIPT_SCHEMA = "livingip.leocleanNoSendIapLocalFailureReceipt.v1"
|
||||||
|
|
@ -944,6 +944,113 @@ def _validate_database_summary(value: Any, request_id: str, expected_name: str)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_installer_transition(value: Any) -> None:
|
||||||
|
_require(
|
||||||
|
isinstance(value, dict)
|
||||||
|
and set(value)
|
||||||
|
== {
|
||||||
|
"outcome",
|
||||||
|
"prior_posture",
|
||||||
|
"prior_n_restarts",
|
||||||
|
"final_posture",
|
||||||
|
"service_action",
|
||||||
|
"managed_files_replaced",
|
||||||
|
}
|
||||||
|
and isinstance(value.get("prior_n_restarts"), int)
|
||||||
|
and not isinstance(value.get("prior_n_restarts"), bool)
|
||||||
|
and value["prior_n_restarts"] >= 0
|
||||||
|
and value.get("final_posture") == "active"
|
||||||
|
and (
|
||||||
|
(
|
||||||
|
value.get("outcome"),
|
||||||
|
value.get("prior_posture"),
|
||||||
|
value.get("service_action"),
|
||||||
|
value.get("managed_files_replaced"),
|
||||||
|
)
|
||||||
|
in {
|
||||||
|
("verified_noop", "active", "none", False),
|
||||||
|
("controlled_start", "inactive", "start", False),
|
||||||
|
("installed", "absent", "restart", True),
|
||||||
|
}
|
||||||
|
or (
|
||||||
|
value.get("outcome") == "replaced"
|
||||||
|
and value.get("prior_posture") in {"active", "inactive"}
|
||||||
|
and value.get("service_action") == "restart"
|
||||||
|
and value.get("managed_files_replaced") is True
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"installer transition was invalid",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_rollback_summary(value: Any) -> str:
|
||||||
|
_require(
|
||||||
|
isinstance(value, dict)
|
||||||
|
and set(value)
|
||||||
|
== {
|
||||||
|
"status",
|
||||||
|
"mode",
|
||||||
|
"executed",
|
||||||
|
"restored_posture",
|
||||||
|
"prior_posture_preserved",
|
||||||
|
},
|
||||||
|
"remote rollback summary was invalid",
|
||||||
|
)
|
||||||
|
installer_automatic = (
|
||||||
|
value.get("status") == "pass"
|
||||||
|
and value.get("mode") == "installer_automatic"
|
||||||
|
and value.get("executed") is True
|
||||||
|
and value.get("restored_posture") in {"active", "inactive", "absent"}
|
||||||
|
and value.get("prior_posture_preserved") is True
|
||||||
|
)
|
||||||
|
explicit = (
|
||||||
|
value.get("status") == "pass"
|
||||||
|
and value.get("mode") == "explicit"
|
||||||
|
and value.get("executed") is True
|
||||||
|
and value.get("restored_posture") in {"active", "inactive", "absent"}
|
||||||
|
and value.get("prior_posture_preserved") is True
|
||||||
|
)
|
||||||
|
posture_recovery = (
|
||||||
|
value.get("status") == "pass"
|
||||||
|
and value.get("mode") == "posture_recovery"
|
||||||
|
and isinstance(value.get("executed"), bool)
|
||||||
|
and value.get("restored_posture") in {"active", "inactive"}
|
||||||
|
and value.get("prior_posture_preserved") is True
|
||||||
|
)
|
||||||
|
not_run = value == {
|
||||||
|
"status": "not_run",
|
||||||
|
"mode": "not_applicable",
|
||||||
|
"executed": False,
|
||||||
|
"restored_posture": "unchanged",
|
||||||
|
"prior_posture_preserved": True,
|
||||||
|
}
|
||||||
|
explicit_failed = (
|
||||||
|
value.get("status") == "fail"
|
||||||
|
and value.get("mode") == "explicit"
|
||||||
|
and isinstance(value.get("executed"), bool)
|
||||||
|
and value.get("restored_posture") == "unknown"
|
||||||
|
and value.get("prior_posture_preserved") is False
|
||||||
|
)
|
||||||
|
posture_recovery_failed = (
|
||||||
|
value.get("status") == "fail"
|
||||||
|
and value.get("mode") == "posture_recovery"
|
||||||
|
and isinstance(value.get("executed"), bool)
|
||||||
|
and value.get("restored_posture") == "unknown"
|
||||||
|
and value.get("prior_posture_preserved") is False
|
||||||
|
)
|
||||||
|
kinds = {
|
||||||
|
"installer_automatic": installer_automatic,
|
||||||
|
"explicit": explicit,
|
||||||
|
"posture_recovery": posture_recovery,
|
||||||
|
"not_run": not_run,
|
||||||
|
"explicit_failed": explicit_failed,
|
||||||
|
"posture_recovery_failed": posture_recovery_failed,
|
||||||
|
}
|
||||||
|
matches = [kind for kind, matched in kinds.items() if matched]
|
||||||
|
_require(len(matches) == 1, "remote rollback summary was invalid")
|
||||||
|
return matches[0]
|
||||||
|
|
||||||
|
|
||||||
def validate_remote_result(plan: Mapping[str, Any], result: Mapping[str, Any]) -> None:
|
def validate_remote_result(plan: Mapping[str, Any], result: Mapping[str, Any]) -> None:
|
||||||
_require(
|
_require(
|
||||||
set(result)
|
set(result)
|
||||||
|
|
@ -999,10 +1106,12 @@ def validate_remote_result(plan: Mapping[str, Any], result: Mapping[str, Any]) -
|
||||||
database = result["database"]
|
database = result["database"]
|
||||||
if operation == "install":
|
if operation == "install":
|
||||||
_require(
|
_require(
|
||||||
isinstance(service, dict) and set(service) == {"name", "before_restart", "after_restart"},
|
isinstance(service, dict)
|
||||||
|
and set(service) == {"name", "installer_transition", "before_restart", "after_restart"},
|
||||||
"install service result was invalid",
|
"install service result was invalid",
|
||||||
)
|
)
|
||||||
_require(service["name"] == SERVICE, "install service target differed")
|
_require(service["name"] == SERVICE, "install service target differed")
|
||||||
|
_validate_installer_transition(service["installer_transition"])
|
||||||
_validate_verification(service["before_restart"], "pre-restart verification")
|
_validate_verification(service["before_restart"], "pre-restart verification")
|
||||||
_validate_verification(service["after_restart"], "post-restart verification")
|
_validate_verification(service["after_restart"], "post-restart verification")
|
||||||
before = service["before_restart"]
|
before = service["before_restart"]
|
||||||
|
|
@ -1079,15 +1188,21 @@ def validate_remote_failure(plan: Mapping[str, Any], result: Mapping[str, Any])
|
||||||
schema = result.get("schema")
|
schema = result.get("schema")
|
||||||
if schema == REMOTE_FAILURE_SCHEMA:
|
if schema == REMOTE_FAILURE_SCHEMA:
|
||||||
_require(set(result) == common_fields, "remote failure fields were not exact")
|
_require(set(result) == common_fields, "remote failure fields were not exact")
|
||||||
|
rollback_kind = _validate_rollback_summary(rollback)
|
||||||
failure_contract = (
|
failure_contract = (
|
||||||
(result.get("failure_stage"), result.get("failure_reason")) in installer.VERIFICATION_FAILURES
|
(result.get("failure_stage"), result.get("failure_reason")) in installer.VERIFICATION_FAILURES
|
||||||
and rollback
|
and (
|
||||||
in (
|
(
|
||||||
{"status": "pass", "mode": "installer_automatic", "restored_posture": "prior"},
|
plan["operation"] == "install"
|
||||||
{"status": "pass", "mode": "explicit", "restored_posture": "active"},
|
and rollback_kind
|
||||||
{"status": "pass", "mode": "explicit", "restored_posture": "inactive"},
|
in {
|
||||||
{"status": "pass", "mode": "explicit", "restored_posture": "absent"},
|
"installer_automatic",
|
||||||
{"status": "not_run", "mode": "not_applicable", "restored_posture": "unchanged"},
|
"explicit",
|
||||||
|
"posture_recovery",
|
||||||
|
"not_run",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
or (plan["operation"] == "verify" and rollback_kind == "not_run")
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
|
|
@ -1096,19 +1211,11 @@ def validate_remote_failure(plan: Mapping[str, Any], result: Mapping[str, Any])
|
||||||
"remote failure fields were not exact",
|
"remote failure fields were not exact",
|
||||||
)
|
)
|
||||||
failure_pair = (result.get("failure_stage"), result.get("failure_reason"))
|
failure_pair = (result.get("failure_stage"), result.get("failure_reason"))
|
||||||
rollback_passed = (
|
rollback_kind = _validate_rollback_summary(rollback)
|
||||||
isinstance(rollback, dict)
|
rollback_passed = rollback_kind in {"explicit", "posture_recovery"}
|
||||||
and set(rollback) == {"status", "mode", "restored_posture"}
|
rollback_failed = rollback_kind in {"explicit_failed", "posture_recovery_failed"}
|
||||||
and rollback.get("status") == "pass"
|
|
||||||
and rollback.get("mode") == "explicit"
|
|
||||||
and rollback.get("restored_posture") in {"active", "inactive", "absent"}
|
|
||||||
)
|
|
||||||
rollback_failed = rollback == {
|
|
||||||
"status": "fail",
|
|
||||||
"mode": "explicit",
|
|
||||||
"restored_posture": "unknown",
|
|
||||||
}
|
|
||||||
fail_closed = result.get("fail_closed")
|
fail_closed = result.get("fail_closed")
|
||||||
|
_require(isinstance(fail_closed, dict), "remote fail-closed summary was invalid")
|
||||||
if rollback_passed:
|
if rollback_passed:
|
||||||
fail_closed_contract = fail_closed == {
|
fail_closed_contract = fail_closed == {
|
||||||
"status": "not_required",
|
"status": "not_required",
|
||||||
|
|
@ -1118,6 +1225,7 @@ def validate_remote_failure(plan: Mapping[str, Any], result: Mapping[str, Any])
|
||||||
"main_pid": None,
|
"main_pid": None,
|
||||||
"n_restarts": None,
|
"n_restarts": None,
|
||||||
"candidate_container": "not_observed",
|
"candidate_container": "not_observed",
|
||||||
|
"service_action_executed": False,
|
||||||
}
|
}
|
||||||
else:
|
else:
|
||||||
fail_closed_contract = (
|
fail_closed_contract = (
|
||||||
|
|
@ -1130,6 +1238,7 @@ def validate_remote_failure(plan: Mapping[str, Any], result: Mapping[str, Any])
|
||||||
"main_pid": None,
|
"main_pid": None,
|
||||||
"n_restarts": None,
|
"n_restarts": None,
|
||||||
"candidate_container": "unknown",
|
"candidate_container": "unknown",
|
||||||
|
"service_action_executed": fail_closed.get("service_action_executed"),
|
||||||
}
|
}
|
||||||
or (
|
or (
|
||||||
isinstance(fail_closed, dict)
|
isinstance(fail_closed, dict)
|
||||||
|
|
@ -1142,6 +1251,7 @@ def validate_remote_failure(plan: Mapping[str, Any], result: Mapping[str, Any])
|
||||||
"main_pid",
|
"main_pid",
|
||||||
"n_restarts",
|
"n_restarts",
|
||||||
"candidate_container",
|
"candidate_container",
|
||||||
|
"service_action_executed",
|
||||||
}
|
}
|
||||||
and fail_closed.get("status") == "pass"
|
and fail_closed.get("status") == "pass"
|
||||||
and fail_closed.get("service_posture") == "inactive"
|
and fail_closed.get("service_posture") == "inactive"
|
||||||
|
|
@ -1152,8 +1262,14 @@ def validate_remote_failure(plan: Mapping[str, Any], result: Mapping[str, Any])
|
||||||
and not isinstance(fail_closed.get("n_restarts"), bool)
|
and not isinstance(fail_closed.get("n_restarts"), bool)
|
||||||
and fail_closed["n_restarts"] >= 0
|
and fail_closed["n_restarts"] >= 0
|
||||||
and fail_closed.get("candidate_container") == "absent"
|
and fail_closed.get("candidate_container") == "absent"
|
||||||
|
and isinstance(fail_closed.get("service_action_executed"), bool)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if isinstance(fail_closed, dict):
|
||||||
|
_require(
|
||||||
|
isinstance(fail_closed.get("service_action_executed"), bool),
|
||||||
|
"remote fail-closed action summary was invalid",
|
||||||
|
)
|
||||||
failure_contract = (
|
failure_contract = (
|
||||||
plan["operation"] == "install"
|
plan["operation"] == "install"
|
||||||
and (
|
and (
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,24 @@ def completed(
|
||||||
return subprocess.CompletedProcess(argv, returncode, stdout, stderr)
|
return subprocess.CompletedProcess(argv, returncode, stdout, stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def install_execution_result(
|
||||||
|
release: dict[str, Any],
|
||||||
|
transition: dict[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema": local.installer.INSTALL_EXECUTION_RESULT_SCHEMA,
|
||||||
|
"status": "pass",
|
||||||
|
"outcome": transition.get("outcome"),
|
||||||
|
"release_sha256": release["release_sha256"],
|
||||||
|
"persisted_receipt": {
|
||||||
|
"schema": local.installer.INSTALL_RECEIPT_SCHEMA,
|
||||||
|
"sha256": "9" * 64,
|
||||||
|
},
|
||||||
|
"transition": transition,
|
||||||
|
"verification": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def private_directory(path: Path) -> Path:
|
def private_directory(path: Path) -> Path:
|
||||||
path.mkdir(mode=0o700)
|
path.mkdir(mode=0o700)
|
||||||
path.chmod(0o700)
|
path.chmod(0o700)
|
||||||
|
|
@ -148,7 +166,9 @@ def verifier_failure_result(
|
||||||
"rollback": {
|
"rollback": {
|
||||||
"status": "pass",
|
"status": "pass",
|
||||||
"mode": "installer_automatic",
|
"mode": "installer_automatic",
|
||||||
"restored_posture": "prior",
|
"executed": True,
|
||||||
|
"restored_posture": "absent",
|
||||||
|
"prior_posture_preserved": True,
|
||||||
},
|
},
|
||||||
"authority": {
|
"authority": {
|
||||||
"sending_enabled": False,
|
"sending_enabled": False,
|
||||||
|
|
@ -169,7 +189,9 @@ def operational_failure_result(plan: dict[str, Any]) -> dict[str, Any]:
|
||||||
result["rollback"] = {
|
result["rollback"] = {
|
||||||
"status": "fail",
|
"status": "fail",
|
||||||
"mode": "explicit",
|
"mode": "explicit",
|
||||||
|
"executed": True,
|
||||||
"restored_posture": "unknown",
|
"restored_posture": "unknown",
|
||||||
|
"prior_posture_preserved": False,
|
||||||
}
|
}
|
||||||
result["fail_closed"] = {
|
result["fail_closed"] = {
|
||||||
"status": "pass",
|
"status": "pass",
|
||||||
|
|
@ -179,6 +201,7 @@ def operational_failure_result(plan: dict[str, Any]) -> dict[str, Any]:
|
||||||
"main_pid": 0,
|
"main_pid": 0,
|
||||||
"n_restarts": 0,
|
"n_restarts": 0,
|
||||||
"candidate_container": "absent",
|
"candidate_container": "absent",
|
||||||
|
"service_action_executed": True,
|
||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
@ -221,6 +244,14 @@ def successful_remote_result(plan: dict[str, Any]) -> dict[str, Any]:
|
||||||
after = verification(103, "5", "6")
|
after = verification(103, "5", "6")
|
||||||
service_result = {
|
service_result = {
|
||||||
"name": local.SERVICE,
|
"name": local.SERVICE,
|
||||||
|
"installer_transition": {
|
||||||
|
"outcome": "installed",
|
||||||
|
"prior_posture": "absent",
|
||||||
|
"prior_n_restarts": 0,
|
||||||
|
"final_posture": "active",
|
||||||
|
"service_action": "restart",
|
||||||
|
"managed_files_replaced": True,
|
||||||
|
},
|
||||||
"before_restart": before,
|
"before_restart": before,
|
||||||
"after_restart": after,
|
"after_restart": after,
|
||||||
}
|
}
|
||||||
|
|
@ -857,10 +888,65 @@ def test_operational_failure_rejects_extra_rollback_fields(tmp_path: Path) -> No
|
||||||
failure = operational_failure_result(plan)
|
failure = operational_failure_result(plan)
|
||||||
failure["rollback"]["credential"] = "not-allowed"
|
failure["rollback"]["credential"] = "not-allowed"
|
||||||
|
|
||||||
with pytest.raises(local.IapRequestError, match="did not match"):
|
with pytest.raises(local.IapRequestError, match="rollback summary was invalid"):
|
||||||
local.validate_remote_failure(plan, failure)
|
local.validate_remote_failure(plan, failure)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"rollback",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"status": "not_run",
|
||||||
|
"mode": "not_applicable",
|
||||||
|
"executed": False,
|
||||||
|
"restored_posture": "unchanged",
|
||||||
|
"prior_posture_preserved": True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"status": "pass",
|
||||||
|
"mode": "installer_automatic",
|
||||||
|
"executed": True,
|
||||||
|
"restored_posture": "inactive",
|
||||||
|
"prior_posture_preserved": True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"status": "pass",
|
||||||
|
"mode": "posture_recovery",
|
||||||
|
"executed": False,
|
||||||
|
"restored_posture": "absent",
|
||||||
|
"prior_posture_preserved": True,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_operational_failure_rejects_impossible_rollback_modes(
|
||||||
|
tmp_path: Path,
|
||||||
|
rollback: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
plan = executable_plan(tmp_path, "install")
|
||||||
|
failure = operational_failure_result(plan)
|
||||||
|
failure["rollback"] = rollback
|
||||||
|
|
||||||
|
with pytest.raises(local.IapRequestError, match=r"remote failure did not match|rollback summary was invalid"):
|
||||||
|
local.validate_remote_failure(plan, failure)
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_failure_rejects_mutating_rollback_summary(tmp_path: Path) -> None:
|
||||||
|
plan = executable_plan(tmp_path, "verify")
|
||||||
|
failure = verifier_failure_result(plan)
|
||||||
|
|
||||||
|
with pytest.raises(local.IapRequestError, match="remote failure did not match"):
|
||||||
|
local.validate_remote_failure(plan, failure)
|
||||||
|
|
||||||
|
failure["rollback"] = {
|
||||||
|
"status": "not_run",
|
||||||
|
"mode": "not_applicable",
|
||||||
|
"executed": False,
|
||||||
|
"restored_posture": "unchanged",
|
||||||
|
"prior_posture_preserved": True,
|
||||||
|
}
|
||||||
|
local.validate_remote_failure(plan, failure)
|
||||||
|
|
||||||
|
|
||||||
def test_remote_main_emits_structured_failure_on_stdout(
|
def test_remote_main_emits_structured_failure_on_stdout(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
|
|
@ -1030,10 +1116,17 @@ def test_outer_lock_and_reused_request_id_fail_closed(tmp_path: Path) -> None:
|
||||||
|
|
||||||
|
|
||||||
def test_rollback_uses_only_explicit_receipt_bound_interface(tmp_path: Path) -> None:
|
def test_rollback_uses_only_explicit_receipt_bound_interface(tmp_path: Path) -> None:
|
||||||
observed: dict[str, Any] = {}
|
observed: list[dict[str, Any]] = []
|
||||||
|
|
||||||
def rollback(release_sha256: str, **kwargs: Any) -> dict[str, Any]:
|
def rollback(release_sha256: str, **kwargs: Any) -> dict[str, Any]:
|
||||||
observed.update({"release_sha256": release_sha256, **kwargs})
|
observed.append({"release_sha256": release_sha256, **kwargs})
|
||||||
|
if kwargs["execute_rollback"] is False:
|
||||||
|
return {
|
||||||
|
"schema": "rollback",
|
||||||
|
"status": "pass",
|
||||||
|
"mode": "dry_run",
|
||||||
|
"expected_release_sha256": release_sha256,
|
||||||
|
}
|
||||||
return {"schema": "rollback", "status": "pass"}
|
return {"schema": "rollback", "status": "pass"}
|
||||||
|
|
||||||
installer = SimpleNamespace(rollback_release=rollback)
|
installer = SimpleNamespace(rollback_release=rollback)
|
||||||
|
|
@ -1046,9 +1139,9 @@ def test_rollback_uses_only_explicit_receipt_bound_interface(tmp_path: Path) ->
|
||||||
)
|
)
|
||||||
|
|
||||||
assert result["status"] == "pass"
|
assert result["status"] == "pass"
|
||||||
assert observed["release_sha256"] == "a" * 64
|
assert [call["release_sha256"] for call in observed] == ["a" * 64, "a" * 64]
|
||||||
assert observed["execute_rollback"] is True
|
assert [call["execute_rollback"] for call in observed] == [False, True]
|
||||||
assert "install_release" not in observed
|
assert all("install_release" not in call for call in observed)
|
||||||
|
|
||||||
|
|
||||||
def test_rollback_fails_when_explicit_interface_is_absent(tmp_path: Path) -> None:
|
def test_rollback_fails_when_explicit_interface_is_absent(tmp_path: Path) -> None:
|
||||||
|
|
@ -1062,6 +1155,26 @@ def test_rollback_fails_when_explicit_interface_is_absent(tmp_path: Path) -> Non
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_rollback_preflight_failure_reports_no_execution() -> None:
|
||||||
|
calls: list[bool] = []
|
||||||
|
|
||||||
|
def rollback(_release_sha256: str, **kwargs: Any) -> dict[str, Any]:
|
||||||
|
calls.append(kwargs["execute_rollback"])
|
||||||
|
return {"schema": "rollback", "status": "fail", "mode": "dry_run"}
|
||||||
|
|
||||||
|
with pytest.raises(remote.ExplicitRollbackFailure) as caught:
|
||||||
|
remote._execute_explicit_rollback(
|
||||||
|
installer_module=SimpleNamespace(rollback_release=rollback),
|
||||||
|
release_sha256="a" * 64,
|
||||||
|
paths=object(),
|
||||||
|
source_binding={"revision": "b" * 40},
|
||||||
|
runner=lambda argv, **_kwargs: completed(argv),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert caught.value.executed is False
|
||||||
|
assert calls == [False]
|
||||||
|
|
||||||
|
|
||||||
def test_install_verifier_failure_returns_bounded_automatic_rollback_result(
|
def test_install_verifier_failure_returns_bounded_automatic_rollback_result(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
|
|
@ -1097,10 +1210,14 @@ def test_install_verifier_failure_returns_bounded_automatic_rollback_result(
|
||||||
return object()
|
return object()
|
||||||
|
|
||||||
def fail_install(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
def fail_install(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
||||||
raise local.installer.VerificationError(
|
failure = local.installer.VerificationError(
|
||||||
"process_environment",
|
"process_environment",
|
||||||
"process_environment_mismatch",
|
"process_environment_mismatch",
|
||||||
)
|
)
|
||||||
|
failure.rollback_executed = True
|
||||||
|
failure.restored_posture = "absent"
|
||||||
|
failure.prior_posture_preserved = True
|
||||||
|
raise failure
|
||||||
|
|
||||||
installer_module = SimpleNamespace(
|
installer_module = SimpleNamespace(
|
||||||
VerificationError=local.installer.VerificationError,
|
VerificationError=local.installer.VerificationError,
|
||||||
|
|
@ -1131,7 +1248,9 @@ def test_install_verifier_failure_returns_bounded_automatic_rollback_result(
|
||||||
assert result["rollback"] == {
|
assert result["rollback"] == {
|
||||||
"status": "pass",
|
"status": "pass",
|
||||||
"mode": "installer_automatic",
|
"mode": "installer_automatic",
|
||||||
"restored_posture": "prior",
|
"executed": True,
|
||||||
|
"restored_posture": "absent",
|
||||||
|
"prior_posture_preserved": True,
|
||||||
}
|
}
|
||||||
assert result["authority"]["sending_enabled"] is False
|
assert result["authority"]["sending_enabled"] is False
|
||||||
|
|
||||||
|
|
@ -1189,26 +1308,47 @@ def test_post_install_proof_failure_executes_explicit_rollback(
|
||||||
def under(_root: Path) -> object:
|
def under(_root: Path) -> object:
|
||||||
return object()
|
return object()
|
||||||
|
|
||||||
|
def rollback_release(release_sha256: str, **kwargs: Any) -> dict[str, Any]:
|
||||||
|
if kwargs["execute_rollback"] is False:
|
||||||
|
return {
|
||||||
|
"schema": "livingip.leocleanNoSendRollbackResult.v1",
|
||||||
|
"status": "pass",
|
||||||
|
"mode": "dry_run",
|
||||||
|
"expected_release_sha256": release_sha256,
|
||||||
|
}
|
||||||
|
rollbacks.append(release_sha256)
|
||||||
|
return {
|
||||||
|
"schema": "livingip.leocleanNoSendRollbackResult.v1",
|
||||||
|
"status": "pass",
|
||||||
|
"mode": "executed",
|
||||||
|
"rolled_back_release_sha256": release_sha256,
|
||||||
|
"restored_release_sha256": None,
|
||||||
|
"restored_service_posture": "absent",
|
||||||
|
"verification": None,
|
||||||
|
}
|
||||||
|
|
||||||
installer_module = SimpleNamespace(
|
installer_module = SimpleNamespace(
|
||||||
SYSTEMCTL="/usr/bin/systemctl",
|
SYSTEMCTL="/usr/bin/systemctl",
|
||||||
VERIFICATION_FAILURES=local.installer.VERIFICATION_FAILURES,
|
VERIFICATION_FAILURES=local.installer.VERIFICATION_FAILURES,
|
||||||
|
INSTALL_EXECUTION_RESULT_SCHEMA=local.installer.INSTALL_EXECUTION_RESULT_SCHEMA,
|
||||||
|
LEGACY_INSTALL_RECEIPT_SCHEMA=local.installer.LEGACY_INSTALL_RECEIPT_SCHEMA,
|
||||||
|
INSTALL_RECEIPT_SCHEMA=local.installer.INSTALL_RECEIPT_SCHEMA,
|
||||||
InstallPaths=Paths,
|
InstallPaths=Paths,
|
||||||
validate_release_bundle=lambda _bundle: (release, b"unit"),
|
validate_release_bundle=lambda _bundle: (release, b"unit"),
|
||||||
_validate_source_binding=lambda _binding: None,
|
_validate_source_binding=lambda _binding: None,
|
||||||
install_release=lambda *_args, **_kwargs: {"status": "pass"},
|
install_release=lambda *_args, **_kwargs: install_execution_result(
|
||||||
verify_running_release=lambda *_args, **_kwargs: next(verifications),
|
release,
|
||||||
rollback_release=lambda release_sha256, **_kwargs: (
|
{
|
||||||
rollbacks.append(release_sha256)
|
"outcome": "installed",
|
||||||
or {
|
"prior_posture": "absent",
|
||||||
"schema": "livingip.leocleanNoSendRollbackResult.v1",
|
"prior_n_restarts": 0,
|
||||||
"status": "pass",
|
"final_posture": "active",
|
||||||
"mode": "executed",
|
"service_action": "restart",
|
||||||
"rolled_back_release_sha256": release_sha256,
|
"managed_files_replaced": True,
|
||||||
"restored_release_sha256": None,
|
},
|
||||||
"restored_service_posture": "absent",
|
|
||||||
"verification": None,
|
|
||||||
}
|
|
||||||
),
|
),
|
||||||
|
verify_running_release=lambda *_args, **_kwargs: next(verifications),
|
||||||
|
rollback_release=rollback_release,
|
||||||
)
|
)
|
||||||
package_module = SimpleNamespace(
|
package_module = SimpleNamespace(
|
||||||
render_systemd_unit=lambda _release: "unit",
|
render_systemd_unit=lambda _release: "unit",
|
||||||
|
|
@ -1256,7 +1396,9 @@ def test_post_install_proof_failure_executes_explicit_rollback(
|
||||||
assert result["rollback"] == {
|
assert result["rollback"] == {
|
||||||
"status": "pass",
|
"status": "pass",
|
||||||
"mode": "explicit",
|
"mode": "explicit",
|
||||||
|
"executed": True,
|
||||||
"restored_posture": "absent",
|
"restored_posture": "absent",
|
||||||
|
"prior_posture_preserved": True,
|
||||||
}
|
}
|
||||||
assert result["fail_closed"] == {
|
assert result["fail_closed"] == {
|
||||||
"status": "not_required",
|
"status": "not_required",
|
||||||
|
|
@ -1266,9 +1408,413 @@ def test_post_install_proof_failure_executes_explicit_rollback(
|
||||||
"main_pid": None,
|
"main_pid": None,
|
||||||
"n_restarts": None,
|
"n_restarts": None,
|
||||||
"candidate_container": "not_observed",
|
"candidate_container": "not_observed",
|
||||||
|
"service_action_executed": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("outcome", "prior_posture", "service_action", "expected_actions", "expected_executed"),
|
||||||
|
[
|
||||||
|
("verified_noop", "active", "none", [], False),
|
||||||
|
("controlled_start", "inactive", "start", ["stop"], True),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_post_install_failure_preserves_exact_release_posture_without_consuming_history(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
outcome: str,
|
||||||
|
prior_posture: str,
|
||||||
|
service_action: str,
|
||||||
|
expected_actions: list[str],
|
||||||
|
expected_executed: bool,
|
||||||
|
) -> None:
|
||||||
|
release = {
|
||||||
|
"release_sha256": "a" * 64,
|
||||||
|
"image": {
|
||||||
|
"reference": f"{remote.IMAGE_REPOSITORY}@sha256:{'b' * 64}",
|
||||||
|
"digest": f"sha256:{'b' * 64}",
|
||||||
|
"config_digest": f"sha256:{'c' * 64}",
|
||||||
|
},
|
||||||
|
"image_input": {"runtime": {"teleo_git_head": "d" * 40}},
|
||||||
|
}
|
||||||
|
manifest = {
|
||||||
|
"operation": "install",
|
||||||
|
"request_id": "iap-abcdefghijkl",
|
||||||
|
"runner_revision": "e" * 40,
|
||||||
|
"release_revision": "d" * 40,
|
||||||
|
"manifest_sha256": "f" * 64,
|
||||||
|
"release": {
|
||||||
|
"release_sha256": release["release_sha256"],
|
||||||
|
"image_reference": release["image"]["reference"],
|
||||||
|
"image_digest": release["image"]["digest"],
|
||||||
|
"image_config_digest": release["image"]["config_digest"],
|
||||||
|
},
|
||||||
|
"release_artifact": {"archive_sha256": "1" * 64},
|
||||||
|
"installer_source": {"revision": "e" * 40},
|
||||||
|
}
|
||||||
|
verification = {
|
||||||
|
"main_pid": 10,
|
||||||
|
"invocation_id": "2" * 32,
|
||||||
|
"container_id": "3" * 64,
|
||||||
|
"n_restarts": 0,
|
||||||
|
}
|
||||||
|
actions: list[str] = []
|
||||||
|
postures = [
|
||||||
|
SimpleNamespace(
|
||||||
|
kind="active",
|
||||||
|
state={"MainPID": 10, "InvocationID": "2" * 32, "NRestarts": 0},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
if prior_posture == "active":
|
||||||
|
postures.append(
|
||||||
|
SimpleNamespace(
|
||||||
|
kind="active",
|
||||||
|
state={"MainPID": 10, "InvocationID": "2" * 32, "NRestarts": 0},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
postures.append(
|
||||||
|
SimpleNamespace(
|
||||||
|
kind="inactive",
|
||||||
|
state={"MainPID": 0, "InvocationID": "", "NRestarts": 0},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
posture_iter = iter(postures)
|
||||||
|
|
||||||
|
class Paths:
|
||||||
|
@staticmethod
|
||||||
|
def under(_root: Path) -> object:
|
||||||
|
return object()
|
||||||
|
|
||||||
|
installer_module = SimpleNamespace(
|
||||||
|
SYSTEMCTL="/usr/bin/systemctl",
|
||||||
|
VERIFICATION_FAILURES=local.installer.VERIFICATION_FAILURES,
|
||||||
|
INSTALL_EXECUTION_RESULT_SCHEMA=local.installer.INSTALL_EXECUTION_RESULT_SCHEMA,
|
||||||
|
LEGACY_INSTALL_RECEIPT_SCHEMA=local.installer.LEGACY_INSTALL_RECEIPT_SCHEMA,
|
||||||
|
INSTALL_RECEIPT_SCHEMA=local.installer.INSTALL_RECEIPT_SCHEMA,
|
||||||
|
InstallPaths=Paths,
|
||||||
|
validate_release_bundle=lambda _bundle: (release, b"unit"),
|
||||||
|
_validate_source_binding=lambda _binding: None,
|
||||||
|
install_release=lambda *_args, **_kwargs: install_execution_result(
|
||||||
|
release,
|
||||||
|
{
|
||||||
|
"outcome": outcome,
|
||||||
|
"prior_posture": prior_posture,
|
||||||
|
"prior_n_restarts": 0,
|
||||||
|
"final_posture": "active",
|
||||||
|
"service_action": service_action,
|
||||||
|
"managed_files_replaced": False,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
verify_running_release=lambda *_args, **_kwargs: verification,
|
||||||
|
rollback_release=lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||||
|
AssertionError("historical rollback must not be consumed")
|
||||||
|
),
|
||||||
|
_systemctl=lambda action, _runner: actions.append(action),
|
||||||
|
_stable_service_posture=lambda *_args, **_kwargs: next(posture_iter),
|
||||||
|
_require_candidate_absent=lambda **_kwargs: None,
|
||||||
|
)
|
||||||
|
package_module = SimpleNamespace(
|
||||||
|
render_systemd_unit=lambda _release: "unit",
|
||||||
|
canonical_sha256=local.package.canonical_sha256,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(remote, "validate_original_release_artifact", lambda *_args, **_kwargs: None)
|
||||||
|
monkeypatch.setattr(remote, "_load_contract_modules", lambda _root: (package_module, installer_module))
|
||||||
|
monkeypatch.setattr(remote, "pull_immutable_image", lambda *_args, **_kwargs: None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
remote,
|
||||||
|
"_verify_database",
|
||||||
|
lambda **_kwargs: (_ for _ in ()).throw(remote.RemoteError("database failed")),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(remote.RemoteOperationFailure) as caught:
|
||||||
|
remote.execute_request(
|
||||||
|
manifest,
|
||||||
|
tmp_path,
|
||||||
|
tmp_path,
|
||||||
|
"6" * 64,
|
||||||
|
"7" * 64,
|
||||||
|
runner=lambda argv, **_kwargs: completed(argv),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = caught.value.result
|
||||||
|
assert actions == expected_actions
|
||||||
|
assert result["rollback"] == {
|
||||||
|
"status": "pass",
|
||||||
|
"mode": "posture_recovery",
|
||||||
|
"executed": expected_executed,
|
||||||
|
"restored_posture": prior_posture,
|
||||||
|
"prior_posture_preserved": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_installer_transition_fails_closed_with_structured_receipt(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
release = {
|
||||||
|
"release_sha256": "a" * 64,
|
||||||
|
"image": {
|
||||||
|
"reference": f"{remote.IMAGE_REPOSITORY}@sha256:{'b' * 64}",
|
||||||
|
"digest": f"sha256:{'b' * 64}",
|
||||||
|
"config_digest": f"sha256:{'c' * 64}",
|
||||||
|
},
|
||||||
|
"image_input": {"runtime": {"teleo_git_head": "d" * 40}},
|
||||||
|
}
|
||||||
|
manifest = {
|
||||||
|
"operation": "install",
|
||||||
|
"request_id": "iap-abcdefghijkl",
|
||||||
|
"runner_revision": "e" * 40,
|
||||||
|
"release_revision": "d" * 40,
|
||||||
|
"manifest_sha256": "f" * 64,
|
||||||
|
"release": {
|
||||||
|
"release_sha256": release["release_sha256"],
|
||||||
|
"image_reference": release["image"]["reference"],
|
||||||
|
"image_digest": release["image"]["digest"],
|
||||||
|
"image_config_digest": release["image"]["config_digest"],
|
||||||
|
},
|
||||||
|
"release_artifact": {"archive_sha256": "1" * 64},
|
||||||
|
"installer_source": {"revision": "e" * 40},
|
||||||
|
}
|
||||||
|
actions: list[str] = []
|
||||||
|
historical_rollbacks: list[bool] = []
|
||||||
|
postures = iter(
|
||||||
|
[
|
||||||
|
SimpleNamespace(
|
||||||
|
kind="active",
|
||||||
|
state={"MainPID": 10, "InvocationID": "2" * 32, "NRestarts": 0},
|
||||||
|
),
|
||||||
|
SimpleNamespace(
|
||||||
|
kind="inactive",
|
||||||
|
state={
|
||||||
|
"ActiveState": "inactive",
|
||||||
|
"SubState": "dead",
|
||||||
|
"MainPID": 0,
|
||||||
|
"InvocationID": "",
|
||||||
|
"NRestarts": 0,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
class Paths:
|
||||||
|
@staticmethod
|
||||||
|
def under(_root: Path) -> object:
|
||||||
|
return object()
|
||||||
|
|
||||||
|
installer_module = SimpleNamespace(
|
||||||
|
SYSTEMCTL="/usr/bin/systemctl",
|
||||||
|
VERIFICATION_FAILURES=local.installer.VERIFICATION_FAILURES,
|
||||||
|
INSTALL_EXECUTION_RESULT_SCHEMA=local.installer.INSTALL_EXECUTION_RESULT_SCHEMA,
|
||||||
|
LEGACY_INSTALL_RECEIPT_SCHEMA=local.installer.LEGACY_INSTALL_RECEIPT_SCHEMA,
|
||||||
|
INSTALL_RECEIPT_SCHEMA=local.installer.INSTALL_RECEIPT_SCHEMA,
|
||||||
|
InstallPaths=Paths,
|
||||||
|
validate_release_bundle=lambda _bundle: (release, b"unit"),
|
||||||
|
_validate_source_binding=lambda _binding: None,
|
||||||
|
install_release=lambda *_args, **_kwargs: install_execution_result(
|
||||||
|
release,
|
||||||
|
{"outcome": "controlled_start"},
|
||||||
|
),
|
||||||
|
verify_running_release=lambda *_args, **_kwargs: {
|
||||||
|
"main_pid": 10,
|
||||||
|
"invocation_id": "2" * 32,
|
||||||
|
"container_id": "3" * 64,
|
||||||
|
"n_restarts": 0,
|
||||||
|
},
|
||||||
|
rollback_release=lambda *_args, **_kwargs: historical_rollbacks.append(True),
|
||||||
|
_systemctl=lambda action, _runner: actions.append(action),
|
||||||
|
_stable_service_posture=lambda *_args, **_kwargs: next(postures),
|
||||||
|
_require_candidate_absent=lambda **_kwargs: None,
|
||||||
|
)
|
||||||
|
package_module = SimpleNamespace(
|
||||||
|
render_systemd_unit=lambda _release: "unit",
|
||||||
|
canonical_sha256=local.package.canonical_sha256,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(remote, "validate_original_release_artifact", lambda *_args, **_kwargs: None)
|
||||||
|
monkeypatch.setattr(remote, "_load_contract_modules", lambda _root: (package_module, installer_module))
|
||||||
|
monkeypatch.setattr(remote, "pull_immutable_image", lambda *_args, **_kwargs: None)
|
||||||
|
|
||||||
|
with pytest.raises(remote.RemoteOperationFailure) as caught:
|
||||||
|
remote.execute_request(
|
||||||
|
manifest,
|
||||||
|
tmp_path,
|
||||||
|
tmp_path,
|
||||||
|
"6" * 64,
|
||||||
|
"7" * 64,
|
||||||
|
runner=lambda argv, **_kwargs: completed(argv),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = caught.value.result
|
||||||
|
assert historical_rollbacks == []
|
||||||
|
assert actions == ["stop"]
|
||||||
|
assert (result["failure_stage"], result["failure_reason"]) == (
|
||||||
|
"install_receipt",
|
||||||
|
"install_receipt_invalid",
|
||||||
|
)
|
||||||
|
assert result["rollback"] == {
|
||||||
|
"status": "fail",
|
||||||
|
"mode": "posture_recovery",
|
||||||
|
"executed": True,
|
||||||
|
"restored_posture": "unknown",
|
||||||
|
"prior_posture_preserved": False,
|
||||||
|
}
|
||||||
|
assert result["fail_closed"]["status"] == "pass"
|
||||||
|
assert result["fail_closed"]["service_action_executed"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_posture_recovery_refuses_identity_drift_before_stop(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
release = {
|
||||||
|
"release_sha256": "a" * 64,
|
||||||
|
"image": {
|
||||||
|
"reference": f"{remote.IMAGE_REPOSITORY}@sha256:{'b' * 64}",
|
||||||
|
"digest": f"sha256:{'b' * 64}",
|
||||||
|
"config_digest": f"sha256:{'c' * 64}",
|
||||||
|
},
|
||||||
|
"image_input": {"runtime": {"teleo_git_head": "d" * 40}},
|
||||||
|
}
|
||||||
|
manifest = {
|
||||||
|
"operation": "install",
|
||||||
|
"request_id": "iap-abcdefghijkl",
|
||||||
|
"runner_revision": "e" * 40,
|
||||||
|
"release_revision": "d" * 40,
|
||||||
|
"manifest_sha256": "f" * 64,
|
||||||
|
"release": {
|
||||||
|
"release_sha256": release["release_sha256"],
|
||||||
|
"image_reference": release["image"]["reference"],
|
||||||
|
"image_digest": release["image"]["digest"],
|
||||||
|
"image_config_digest": release["image"]["config_digest"],
|
||||||
|
},
|
||||||
|
"release_artifact": {"archive_sha256": "1" * 64},
|
||||||
|
"installer_source": {"revision": "e" * 40},
|
||||||
|
}
|
||||||
|
actions: list[str] = []
|
||||||
|
postures = iter(
|
||||||
|
[
|
||||||
|
SimpleNamespace(kind="active", state={"MainPID": 10, "InvocationID": "2" * 32, "NRestarts": 0}),
|
||||||
|
SimpleNamespace(kind="active", state={"MainPID": 10, "InvocationID": "2" * 32, "NRestarts": 0}),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
verifications = iter(
|
||||||
|
[
|
||||||
|
{"main_pid": 10, "invocation_id": "2" * 32, "container_id": "3" * 64, "n_restarts": 0},
|
||||||
|
{"main_pid": 99, "invocation_id": "9" * 32, "container_id": "8" * 64, "n_restarts": 0},
|
||||||
|
{"main_pid": 99, "invocation_id": "9" * 32, "container_id": "8" * 64, "n_restarts": 0},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
class Paths:
|
||||||
|
@staticmethod
|
||||||
|
def under(_root: Path) -> object:
|
||||||
|
return object()
|
||||||
|
|
||||||
|
installer_module = SimpleNamespace(
|
||||||
|
SYSTEMCTL="/usr/bin/systemctl",
|
||||||
|
VERIFICATION_FAILURES=local.installer.VERIFICATION_FAILURES,
|
||||||
|
INSTALL_EXECUTION_RESULT_SCHEMA=local.installer.INSTALL_EXECUTION_RESULT_SCHEMA,
|
||||||
|
LEGACY_INSTALL_RECEIPT_SCHEMA=local.installer.LEGACY_INSTALL_RECEIPT_SCHEMA,
|
||||||
|
INSTALL_RECEIPT_SCHEMA=local.installer.INSTALL_RECEIPT_SCHEMA,
|
||||||
|
InstallPaths=Paths,
|
||||||
|
validate_release_bundle=lambda _bundle: (release, b"unit"),
|
||||||
|
_validate_source_binding=lambda _binding: None,
|
||||||
|
install_release=lambda *_args, **_kwargs: install_execution_result(
|
||||||
|
release,
|
||||||
|
{
|
||||||
|
"outcome": "controlled_start",
|
||||||
|
"prior_posture": "inactive",
|
||||||
|
"prior_n_restarts": 0,
|
||||||
|
"final_posture": "active",
|
||||||
|
"service_action": "start",
|
||||||
|
"managed_files_replaced": False,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
verify_running_release=lambda *_args, **_kwargs: next(verifications),
|
||||||
|
rollback_release=lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||||
|
AssertionError("historical rollback must not be consumed")
|
||||||
|
),
|
||||||
|
_systemctl=lambda action, _runner: actions.append(action),
|
||||||
|
_stable_service_posture=lambda *_args, **_kwargs: next(postures),
|
||||||
|
_require_candidate_absent=lambda **_kwargs: None,
|
||||||
|
)
|
||||||
|
package_module = SimpleNamespace(
|
||||||
|
render_systemd_unit=lambda _release: "unit",
|
||||||
|
canonical_sha256=local.package.canonical_sha256,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(remote, "validate_original_release_artifact", lambda *_args, **_kwargs: None)
|
||||||
|
monkeypatch.setattr(remote, "_load_contract_modules", lambda _root: (package_module, installer_module))
|
||||||
|
monkeypatch.setattr(remote, "pull_immutable_image", lambda *_args, **_kwargs: None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
remote,
|
||||||
|
"_verify_database",
|
||||||
|
lambda **_kwargs: (_ for _ in ()).throw(remote.RemoteError("database failed")),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(remote.RemoteOperationFailure) as caught:
|
||||||
|
remote.execute_request(
|
||||||
|
manifest,
|
||||||
|
tmp_path,
|
||||||
|
tmp_path,
|
||||||
|
"6" * 64,
|
||||||
|
"7" * 64,
|
||||||
|
runner=lambda argv, **_kwargs: completed(argv),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = caught.value.result
|
||||||
|
assert actions == []
|
||||||
|
assert result["rollback"]["executed"] is False
|
||||||
|
assert result["rollback"]["mode"] == "posture_recovery"
|
||||||
|
assert result["fail_closed"]["status"] == "fail"
|
||||||
|
assert result["fail_closed"]["service_action_executed"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_posture_recovery_rejects_restart_counter_drift_after_fixed_restart() -> None:
|
||||||
|
actions: list[str] = []
|
||||||
|
postures = iter(
|
||||||
|
[
|
||||||
|
SimpleNamespace(
|
||||||
|
kind="active",
|
||||||
|
state={"MainPID": 10, "InvocationID": "2" * 32, "NRestarts": 1},
|
||||||
|
),
|
||||||
|
SimpleNamespace(
|
||||||
|
kind="inactive",
|
||||||
|
state={"MainPID": 0, "InvocationID": "", "NRestarts": 1},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
installer_module = SimpleNamespace(
|
||||||
|
_systemctl=lambda action, _runner: actions.append(action),
|
||||||
|
_stable_service_posture=lambda *_args, **_kwargs: next(postures),
|
||||||
|
verify_running_release=lambda *_args, **_kwargs: {
|
||||||
|
"main_pid": 10,
|
||||||
|
"invocation_id": "2" * 32,
|
||||||
|
"container_id": "3" * 64,
|
||||||
|
"n_restarts": 1,
|
||||||
|
},
|
||||||
|
_require_candidate_absent=lambda **_kwargs: None,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(remote.PostureRecoveryFailure) as caught:
|
||||||
|
remote._restore_transition_posture(
|
||||||
|
installer_module=installer_module,
|
||||||
|
release={"release_sha256": "a" * 64},
|
||||||
|
transition={
|
||||||
|
"outcome": "controlled_start",
|
||||||
|
"prior_posture": "inactive",
|
||||||
|
"prior_n_restarts": 0,
|
||||||
|
"final_posture": "active",
|
||||||
|
"service_action": "start",
|
||||||
|
"managed_files_replaced": False,
|
||||||
|
},
|
||||||
|
paths=object(),
|
||||||
|
runner=lambda argv, **_kwargs: completed(argv),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert caught.value.executed is True
|
||||||
|
assert actions == ["stop"]
|
||||||
|
|
||||||
|
|
||||||
def test_incomplete_rollback_stops_service_and_reports_inert_posture(
|
def test_incomplete_rollback_stops_service_and_reports_inert_posture(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
|
|
@ -1329,19 +1875,43 @@ def test_incomplete_rollback_stops_service_and_reports_inert_posture(
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def fail_rollback(release_sha256: str, **kwargs: Any) -> dict[str, Any]:
|
||||||
|
if kwargs["execute_rollback"] is False:
|
||||||
|
return {
|
||||||
|
"schema": "livingip.leocleanNoSendRollbackResult.v1",
|
||||||
|
"status": "pass",
|
||||||
|
"mode": "dry_run",
|
||||||
|
"expected_release_sha256": release_sha256,
|
||||||
|
}
|
||||||
|
raise RuntimeError("rollback failed")
|
||||||
|
|
||||||
installer_module = SimpleNamespace(
|
installer_module = SimpleNamespace(
|
||||||
SYSTEMCTL="/usr/bin/systemctl",
|
SYSTEMCTL="/usr/bin/systemctl",
|
||||||
|
INSTALL_EXECUTION_RESULT_SCHEMA=local.installer.INSTALL_EXECUTION_RESULT_SCHEMA,
|
||||||
|
LEGACY_INSTALL_RECEIPT_SCHEMA=local.installer.LEGACY_INSTALL_RECEIPT_SCHEMA,
|
||||||
|
INSTALL_RECEIPT_SCHEMA=local.installer.INSTALL_RECEIPT_SCHEMA,
|
||||||
InstallPaths=Paths,
|
InstallPaths=Paths,
|
||||||
validate_release_bundle=lambda _bundle: (release, b"unit"),
|
validate_release_bundle=lambda _bundle: (release, b"unit"),
|
||||||
_validate_source_binding=lambda _binding: None,
|
_validate_source_binding=lambda _binding: None,
|
||||||
install_release=lambda *_args, **_kwargs: {"status": "pass"},
|
install_release=lambda *_args, **_kwargs: install_execution_result(
|
||||||
|
release,
|
||||||
|
{
|
||||||
|
"outcome": "installed",
|
||||||
|
"prior_posture": "absent",
|
||||||
|
"prior_n_restarts": 0,
|
||||||
|
"final_posture": "active",
|
||||||
|
"service_action": "restart",
|
||||||
|
"managed_files_replaced": True,
|
||||||
|
},
|
||||||
|
),
|
||||||
verify_running_release=lambda *_args, **_kwargs: {
|
verify_running_release=lambda *_args, **_kwargs: {
|
||||||
"main_pid": 10,
|
"main_pid": 10,
|
||||||
"invocation_id": "2" * 32,
|
"invocation_id": "2" * 32,
|
||||||
"container_id": "3" * 64,
|
"container_id": "3" * 64,
|
||||||
"n_restarts": 0,
|
"n_restarts": 0,
|
||||||
},
|
},
|
||||||
rollback_release=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("rollback failed")),
|
rollback_release=fail_rollback,
|
||||||
_systemctl=lambda action, _runner: stops.append(action),
|
_systemctl=lambda action, _runner: stops.append(action),
|
||||||
_stable_service_posture=lambda *_args, **_kwargs: next(postures),
|
_stable_service_posture=lambda *_args, **_kwargs: next(postures),
|
||||||
_require_candidate_absent=lambda **_kwargs: absence_checks.append(True),
|
_require_candidate_absent=lambda **_kwargs: absence_checks.append(True),
|
||||||
|
|
@ -1377,7 +1947,9 @@ def test_incomplete_rollback_stops_service_and_reports_inert_posture(
|
||||||
assert result["rollback"] == {
|
assert result["rollback"] == {
|
||||||
"status": "fail",
|
"status": "fail",
|
||||||
"mode": "explicit",
|
"mode": "explicit",
|
||||||
|
"executed": True,
|
||||||
"restored_posture": "unknown",
|
"restored_posture": "unknown",
|
||||||
|
"prior_posture_preserved": False,
|
||||||
}
|
}
|
||||||
assert result["fail_closed"] == {
|
assert result["fail_closed"] == {
|
||||||
"status": "pass",
|
"status": "pass",
|
||||||
|
|
@ -1387,6 +1959,7 @@ def test_incomplete_rollback_stops_service_and_reports_inert_posture(
|
||||||
"main_pid": 0,
|
"main_pid": 0,
|
||||||
"n_restarts": 0,
|
"n_restarts": 0,
|
||||||
"candidate_container": "absent",
|
"candidate_container": "absent",
|
||||||
|
"service_action_executed": True,
|
||||||
}
|
}
|
||||||
assert result["authority"]["sending_enabled"] is False
|
assert result["authority"]["sending_enabled"] is False
|
||||||
|
|
||||||
|
|
@ -1418,6 +1991,7 @@ def test_fail_closed_stop_refuses_untrusted_unit_or_dropin() -> None:
|
||||||
"main_pid": None,
|
"main_pid": None,
|
||||||
"n_restarts": None,
|
"n_restarts": None,
|
||||||
"candidate_container": "unknown",
|
"candidate_container": "unknown",
|
||||||
|
"service_action_executed": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -110,6 +110,7 @@ class FakeHost:
|
||||||
self.current_release: dict[str, object] | None = None
|
self.current_release: dict[str, object] | None = None
|
||||||
self.loaded_release: dict[str, object] | None = None
|
self.loaded_release: dict[str, object] | None = None
|
||||||
self.generation = 0
|
self.generation = 0
|
||||||
|
self.start_calls = 0
|
||||||
self.restart_calls = 0
|
self.restart_calls = 0
|
||||||
self.stop_calls = 0
|
self.stop_calls = 0
|
||||||
self.fail_dropin_release: str | None = None
|
self.fail_dropin_release: str | None = None
|
||||||
|
|
@ -157,7 +158,7 @@ class FakeHost:
|
||||||
(directory / "cmdline").write_bytes(b"\0".join(item.encode() for item in command) + b"\0")
|
(directory / "cmdline").write_bytes(b"\0".join(item.encode() for item in command) + b"\0")
|
||||||
(directory / "cgroup").write_text(f"0::/system.slice/{package.SERVICE}\n", encoding="utf-8")
|
(directory / "cgroup").write_text(f"0::/system.slice/{package.SERVICE}\n", encoding="utf-8")
|
||||||
|
|
||||||
def _start(self) -> None:
|
def _start(self, action: str = "restart") -> None:
|
||||||
release = self.loaded_release
|
release = self.loaded_release
|
||||||
assert release is not None
|
assert release is not None
|
||||||
image = release["image"]
|
image = release["image"]
|
||||||
|
|
@ -166,7 +167,10 @@ class FakeHost:
|
||||||
runtime = inspection["runtime_config"]
|
runtime = inspection["runtime_config"]
|
||||||
assert isinstance(runtime, dict)
|
assert isinstance(runtime, dict)
|
||||||
self.generation += 1
|
self.generation += 1
|
||||||
self.restart_calls += 1
|
if action == "start":
|
||||||
|
self.start_calls += 1
|
||||||
|
else:
|
||||||
|
self.restart_calls += 1
|
||||||
self.active = True
|
self.active = True
|
||||||
self.main_pid = 1000 + self.generation
|
self.main_pid = 1000 + self.generation
|
||||||
self.invocation = f"{self.generation:032x}"
|
self.invocation = f"{self.generation:032x}"
|
||||||
|
|
@ -274,6 +278,9 @@ class FakeHost:
|
||||||
if action == "restart":
|
if action == "restart":
|
||||||
self._start()
|
self._start()
|
||||||
return subprocess.CompletedProcess(arguments, 0, stdout=b"", stderr=b"")
|
return subprocess.CompletedProcess(arguments, 0, stdout=b"", stderr=b"")
|
||||||
|
if action == "start":
|
||||||
|
self._start("start")
|
||||||
|
return subprocess.CompletedProcess(arguments, 0, stdout=b"", stderr=b"")
|
||||||
if action == "stop":
|
if action == "stop":
|
||||||
self._stop()
|
self._stop()
|
||||||
return subprocess.CompletedProcess(arguments, 0, stdout=b"", stderr=b"")
|
return subprocess.CompletedProcess(arguments, 0, stdout=b"", stderr=b"")
|
||||||
|
|
@ -397,7 +404,7 @@ def test_disposable_root_install_is_digest_bound_idempotent_and_exact(tmp_path:
|
||||||
host = FakeHost(paths)
|
host = FakeHost(paths)
|
||||||
host.add_release(release, image)
|
host.add_release(release, image)
|
||||||
|
|
||||||
receipt = installer.install_release(
|
result = installer.install_release(
|
||||||
bundle,
|
bundle,
|
||||||
paths=paths,
|
paths=paths,
|
||||||
source_binding=source_binding(),
|
source_binding=source_binding(),
|
||||||
|
|
@ -405,10 +412,26 @@ def test_disposable_root_install_is_digest_bound_idempotent_and_exact(tmp_path:
|
||||||
sleeper=lambda _seconds: None,
|
sleeper=lambda _seconds: None,
|
||||||
execute_restart=True,
|
execute_restart=True,
|
||||||
)
|
)
|
||||||
|
receipt = json.loads(paths.receipt.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
assert receipt["status"] == "pass"
|
assert result["status"] == "pass"
|
||||||
assert receipt["schema"] == "livingip.leocleanNoSendInstallReceipt.v2"
|
assert result["schema"] == installer.INSTALL_EXECUTION_RESULT_SCHEMA
|
||||||
assert receipt["release_sha256"] == release["release_sha256"]
|
assert result["outcome"] == "installed"
|
||||||
|
assert result["transition"] == {
|
||||||
|
"outcome": "installed",
|
||||||
|
"prior_posture": "absent",
|
||||||
|
"prior_n_restarts": 0,
|
||||||
|
"final_posture": "active",
|
||||||
|
"service_action": "restart",
|
||||||
|
"managed_files_replaced": True,
|
||||||
|
}
|
||||||
|
assert result["release_sha256"] == release["release_sha256"]
|
||||||
|
assert result["persisted_receipt"] == {
|
||||||
|
"schema": installer.INSTALL_RECEIPT_SCHEMA,
|
||||||
|
"sha256": package.sha256_file(paths.receipt),
|
||||||
|
}
|
||||||
|
assert receipt["schema"] == installer.INSTALL_RECEIPT_SCHEMA
|
||||||
|
assert receipt["outcome"] == "installed"
|
||||||
assert host.restart_calls == 1
|
assert host.restart_calls == 1
|
||||||
assert paths.unit.read_text(encoding="utf-8") == package.render_systemd_unit(release)
|
assert paths.unit.read_text(encoding="utf-8") == package.render_systemd_unit(release)
|
||||||
assert paths.unit.stat().st_mode & 0o777 == 0o644
|
assert paths.unit.stat().st_mode & 0o777 == 0o644
|
||||||
|
|
@ -423,6 +446,12 @@ def test_disposable_root_install_is_digest_bound_idempotent_and_exact(tmp_path:
|
||||||
assert not list(paths.docker_config.iterdir())
|
assert not list(paths.docker_config.iterdir())
|
||||||
assert not list(paths.unit.parent.glob(f".{paths.unit.name}.*.tmp"))
|
assert not list(paths.unit.parent.glob(f".{paths.unit.name}.*.tmp"))
|
||||||
|
|
||||||
|
managed_before = {
|
||||||
|
role: path.read_bytes()
|
||||||
|
for role, (path, _mode) in installer._managed_files(paths).items()
|
||||||
|
}
|
||||||
|
identity_before = (host.main_pid, host.n_restarts, host.invocation)
|
||||||
|
calls_before = len(host.calls)
|
||||||
repeated = installer.install_release(
|
repeated = installer.install_release(
|
||||||
bundle,
|
bundle,
|
||||||
paths=paths,
|
paths=paths,
|
||||||
|
|
@ -432,8 +461,31 @@ def test_disposable_root_install_is_digest_bound_idempotent_and_exact(tmp_path:
|
||||||
execute_restart=True,
|
execute_restart=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert repeated["result"] == "already_installed"
|
assert repeated["outcome"] == "verified_noop"
|
||||||
|
assert repeated["schema"] == installer.INSTALL_EXECUTION_RESULT_SCHEMA
|
||||||
|
assert repeated["persisted_receipt"] == {
|
||||||
|
"schema": installer.INSTALL_RECEIPT_SCHEMA,
|
||||||
|
"sha256": package.sha256_file(paths.receipt),
|
||||||
|
}
|
||||||
|
assert repeated["transition"] == {
|
||||||
|
"outcome": "verified_noop",
|
||||||
|
"prior_posture": "active",
|
||||||
|
"prior_n_restarts": 0,
|
||||||
|
"final_posture": "active",
|
||||||
|
"service_action": "none",
|
||||||
|
"managed_files_replaced": False,
|
||||||
|
}
|
||||||
assert host.restart_calls == 1
|
assert host.restart_calls == 1
|
||||||
|
assert host.start_calls == 0
|
||||||
|
assert (host.main_pid, host.n_restarts, host.invocation) == identity_before
|
||||||
|
assert {
|
||||||
|
role: path.read_bytes()
|
||||||
|
for role, (path, _mode) in installer._managed_files(paths).items()
|
||||||
|
} == managed_before
|
||||||
|
assert not any(
|
||||||
|
call[0] == installer.SYSTEMCTL and call[1] in {"start", "restart", "stop", "daemon-reload"}
|
||||||
|
for call in host.calls[calls_before:]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_first_install_accepts_production_shaped_systemd_not_found_defaults(tmp_path: Path) -> None:
|
def test_first_install_accepts_production_shaped_systemd_not_found_defaults(tmp_path: Path) -> None:
|
||||||
|
|
@ -465,10 +517,391 @@ def test_first_install_accepts_production_shaped_systemd_not_found_defaults(tmp_
|
||||||
execute_restart=True,
|
execute_restart=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert repeated["result"] == "already_installed"
|
assert repeated["outcome"] == "verified_noop"
|
||||||
assert host.restart_calls == 1
|
assert host.restart_calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_exact_inactive_release_converges_with_one_controlled_start_and_preserves_history(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
bundle, release, image = release_bundle(tmp_path, "2")
|
||||||
|
paths = install_paths(tmp_path)
|
||||||
|
host = FakeHost(paths)
|
||||||
|
host.add_release(release, image)
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
host._stop()
|
||||||
|
managed_before = {
|
||||||
|
role: path.read_bytes()
|
||||||
|
for role, (path, _mode) in installer._managed_files(paths).items()
|
||||||
|
}
|
||||||
|
calls_before = len(host.calls)
|
||||||
|
restart_calls = host.restart_calls
|
||||||
|
stop_calls = host.stop_calls
|
||||||
|
|
||||||
|
result = installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["outcome"] == "controlled_start"
|
||||||
|
assert result["schema"] == installer.INSTALL_EXECUTION_RESULT_SCHEMA
|
||||||
|
assert result["transition"] == {
|
||||||
|
"outcome": "controlled_start",
|
||||||
|
"prior_posture": "inactive",
|
||||||
|
"prior_n_restarts": 0,
|
||||||
|
"final_posture": "active",
|
||||||
|
"service_action": "start",
|
||||||
|
"managed_files_replaced": False,
|
||||||
|
}
|
||||||
|
assert host.active
|
||||||
|
assert host.start_calls == 1
|
||||||
|
assert host.restart_calls == restart_calls
|
||||||
|
assert host.stop_calls == stop_calls
|
||||||
|
assert {
|
||||||
|
role: path.read_bytes()
|
||||||
|
for role, (path, _mode) in installer._managed_files(paths).items()
|
||||||
|
} == managed_before
|
||||||
|
assert [
|
||||||
|
call[1]
|
||||||
|
for call in host.calls[calls_before:]
|
||||||
|
if call[0] == installer.SYSTEMCTL and call[1] in {"start", "restart", "stop", "daemon-reload"}
|
||||||
|
] == ["start"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"role",
|
||||||
|
["unit", "helper", "release", "receipt", "rollback_state"],
|
||||||
|
)
|
||||||
|
def test_each_missing_managed_file_is_rejected_as_partial_before_mutation(
|
||||||
|
tmp_path: Path,
|
||||||
|
role: str,
|
||||||
|
) -> None:
|
||||||
|
bundle, release, image = release_bundle(tmp_path, "2")
|
||||||
|
paths = install_paths(tmp_path)
|
||||||
|
host = FakeHost(paths)
|
||||||
|
host.add_release(release, image)
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
managed_path = installer._managed_files(paths)[role][0]
|
||||||
|
managed_path.unlink()
|
||||||
|
calls_before = len(host.calls)
|
||||||
|
|
||||||
|
with pytest.raises(installer.InstallError, match="managed install state was partial"):
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not any(
|
||||||
|
call[0] == installer.SYSTEMCTL and call[1] in {"start", "restart", "stop", "daemon-reload"}
|
||||||
|
for call in host.calls[calls_before:]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"mutation",
|
||||||
|
[
|
||||||
|
"outer_extra",
|
||||||
|
"rollback_extra",
|
||||||
|
"previous_release_integer",
|
||||||
|
"previous_active_integer",
|
||||||
|
"automatic_integer",
|
||||||
|
"service_restart_drift",
|
||||||
|
"rollback_posture_drift",
|
||||||
|
"load_state_drift",
|
||||||
|
"main_pid_drift",
|
||||||
|
"verification_forged",
|
||||||
|
"authority_forged",
|
||||||
|
"invocation_integer",
|
||||||
|
"container_integer",
|
||||||
|
"cmdline_integer",
|
||||||
|
"environment_nonstring",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_current_install_receipt_is_exact_and_bound_to_rollback_state(
|
||||||
|
tmp_path: Path,
|
||||||
|
mutation: str,
|
||||||
|
) -> None:
|
||||||
|
bundle, release, image = release_bundle(tmp_path, "2")
|
||||||
|
paths = install_paths(tmp_path)
|
||||||
|
host = FakeHost(paths)
|
||||||
|
host.add_release(release, image)
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
receipt = json.loads(paths.receipt.read_text(encoding="utf-8"))
|
||||||
|
if mutation == "outer_extra":
|
||||||
|
receipt["unexpected"] = "field"
|
||||||
|
elif mutation == "rollback_extra":
|
||||||
|
receipt["rollback"]["unexpected"] = "field"
|
||||||
|
elif mutation == "previous_release_integer":
|
||||||
|
receipt["rollback"]["previous_release_present"] = 0
|
||||||
|
elif mutation == "previous_active_integer":
|
||||||
|
receipt["rollback"]["previous_service_active"] = 0
|
||||||
|
elif mutation == "automatic_integer":
|
||||||
|
receipt["rollback"]["automatic_on_failure"] = 1
|
||||||
|
elif mutation == "service_restart_drift":
|
||||||
|
receipt["service_before"]["n_restarts"] += 1
|
||||||
|
elif mutation == "rollback_posture_drift":
|
||||||
|
receipt["rollback"]["previous_service_posture"] = "active"
|
||||||
|
elif mutation == "load_state_drift":
|
||||||
|
receipt["service_before"]["load_state"] = "loaded"
|
||||||
|
elif mutation == "main_pid_drift":
|
||||||
|
receipt["service_before"]["main_pid"] = 12345
|
||||||
|
elif mutation == "verification_forged":
|
||||||
|
receipt["verification"] = {"status": "forged"}
|
||||||
|
elif mutation == "authority_forged":
|
||||||
|
receipt["authority"] = {"cloud_or_database_authority_changed": True}
|
||||||
|
elif mutation == "invocation_integer":
|
||||||
|
receipt["verification"]["invocation_id"] = int("1" * 32)
|
||||||
|
elif mutation == "container_integer":
|
||||||
|
receipt["verification"]["container_id"] = int("2" * 64)
|
||||||
|
elif mutation == "cmdline_integer":
|
||||||
|
receipt["verification"]["supervising_process"]["cmdline_sha256"] = int("3" * 64)
|
||||||
|
else:
|
||||||
|
receipt["verification"]["validated_process_environment_fields"] = [{}]
|
||||||
|
paths.receipt.write_text(json.dumps(receipt, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
paths.receipt.chmod(0o600)
|
||||||
|
stop_calls = host.stop_calls
|
||||||
|
|
||||||
|
with pytest.raises(installer.InstallError, match="installed receipt"):
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
with pytest.raises(installer.InstallError, match="installed receipt"):
|
||||||
|
installer._validate_managed_snapshots(
|
||||||
|
installer._current_managed_snapshots(paths),
|
||||||
|
paths=paths,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert host.stop_calls == stop_calls
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_v2_install_receipt_remains_readable_without_rewrite(tmp_path: Path) -> None:
|
||||||
|
bundle, release, image = release_bundle(tmp_path, "2")
|
||||||
|
paths = install_paths(tmp_path)
|
||||||
|
host = FakeHost(paths)
|
||||||
|
host.add_release(release, image)
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
legacy = json.loads(paths.receipt.read_text(encoding="utf-8"))
|
||||||
|
legacy["schema"] = installer.LEGACY_INSTALL_RECEIPT_SCHEMA
|
||||||
|
legacy["result"] = "installed"
|
||||||
|
legacy.pop("outcome")
|
||||||
|
legacy.pop("transition")
|
||||||
|
paths.receipt.write_text(json.dumps(legacy, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
paths.receipt.chmod(0o600)
|
||||||
|
receipt_before = paths.receipt.read_bytes()
|
||||||
|
|
||||||
|
result = installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["outcome"] == "verified_noop"
|
||||||
|
assert paths.receipt.read_bytes() == receipt_before
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("failure_mode", ["start_command", "post_start_verification"])
|
||||||
|
def test_exact_inactive_start_failure_restores_inactive_without_rewriting_history(
|
||||||
|
tmp_path: Path,
|
||||||
|
failure_mode: str,
|
||||||
|
) -> None:
|
||||||
|
bundle, release, image = release_bundle(tmp_path, "2")
|
||||||
|
paths = install_paths(tmp_path)
|
||||||
|
host = FakeHost(paths)
|
||||||
|
host.add_release(release, image)
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
host._stop()
|
||||||
|
managed_before = {
|
||||||
|
role: path.read_bytes()
|
||||||
|
for role, (path, _mode) in installer._managed_files(paths).items()
|
||||||
|
}
|
||||||
|
if failure_mode == "post_start_verification":
|
||||||
|
host.fail_dropin_release = str(release["release_sha256"])
|
||||||
|
|
||||||
|
def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]:
|
||||||
|
completed = host(arguments)
|
||||||
|
if failure_mode == "start_command" and arguments[:2] == [installer.SYSTEMCTL, "start"]:
|
||||||
|
return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=b"injected")
|
||||||
|
return completed
|
||||||
|
|
||||||
|
with pytest.raises(installer.VerificationError) as caught:
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=runner,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert caught.value.rollback_executed is True
|
||||||
|
assert caught.value.restored_posture == "inactive"
|
||||||
|
assert caught.value.prior_posture_preserved is True
|
||||||
|
assert not host.active
|
||||||
|
assert host.container is None
|
||||||
|
assert host.start_calls == 1
|
||||||
|
assert {
|
||||||
|
role: path.read_bytes()
|
||||||
|
for role, (path, _mode) in installer._managed_files(paths).items()
|
||||||
|
} == managed_before
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("cleanup_failure", ["stop_command", "restart_counter_drift"])
|
||||||
|
def test_exact_inactive_cleanup_failure_never_claims_prior_posture_preserved(
|
||||||
|
tmp_path: Path,
|
||||||
|
cleanup_failure: str,
|
||||||
|
) -> None:
|
||||||
|
bundle, release, image = release_bundle(tmp_path, "2")
|
||||||
|
paths = install_paths(tmp_path)
|
||||||
|
host = FakeHost(paths)
|
||||||
|
host.add_release(release, image)
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
host._stop()
|
||||||
|
host.fail_dropin_release = str(release["release_sha256"])
|
||||||
|
managed_before = {
|
||||||
|
role: path.read_bytes()
|
||||||
|
for role, (path, _mode) in installer._managed_files(paths).items()
|
||||||
|
}
|
||||||
|
|
||||||
|
def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]:
|
||||||
|
completed = host(arguments)
|
||||||
|
if arguments[:2] == [installer.SYSTEMCTL, "stop"]:
|
||||||
|
if cleanup_failure == "stop_command":
|
||||||
|
return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=b"injected")
|
||||||
|
host.n_restarts += 1
|
||||||
|
return completed
|
||||||
|
|
||||||
|
with pytest.raises(installer.InstallError, match="cleanup was incomplete") as caught:
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=runner,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert getattr(caught.value, "prior_posture_preserved", None) is not True
|
||||||
|
assert not host.active
|
||||||
|
assert {
|
||||||
|
role: path.read_bytes()
|
||||||
|
for role, (path, _mode) in installer._managed_files(paths).items()
|
||||||
|
} == managed_before
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("prior_posture", ["active", "inactive"])
|
||||||
|
def test_exact_release_rejects_receipt_replacement_race(
|
||||||
|
tmp_path: Path,
|
||||||
|
prior_posture: str,
|
||||||
|
) -> None:
|
||||||
|
bundle, release, image = release_bundle(tmp_path, "2")
|
||||||
|
paths = install_paths(tmp_path)
|
||||||
|
host = FakeHost(paths)
|
||||||
|
host.add_release(release, image)
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
if prior_posture == "inactive":
|
||||||
|
host._stop()
|
||||||
|
replaced = False
|
||||||
|
service_show_calls = 0
|
||||||
|
|
||||||
|
def replace_receipt() -> None:
|
||||||
|
nonlocal replaced
|
||||||
|
if replaced:
|
||||||
|
return
|
||||||
|
replaced = True
|
||||||
|
paths.receipt.unlink()
|
||||||
|
paths.receipt.write_text('{"schema":"forged"}\n', encoding="utf-8")
|
||||||
|
paths.receipt.chmod(0o600)
|
||||||
|
|
||||||
|
def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]:
|
||||||
|
nonlocal service_show_calls
|
||||||
|
completed = host(arguments)
|
||||||
|
if prior_posture == "active" and arguments[:2] == [installer.SYSTEMCTL, "show"]:
|
||||||
|
service_show_calls += 1
|
||||||
|
if service_show_calls == 3:
|
||||||
|
replace_receipt()
|
||||||
|
if prior_posture == "inactive" and arguments[:2] == [installer.SYSTEMCTL, "start"]:
|
||||||
|
replace_receipt()
|
||||||
|
return completed
|
||||||
|
|
||||||
|
with pytest.raises(installer.InstallError, match="file identity drifted"):
|
||||||
|
installer.install_release(
|
||||||
|
bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=runner,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert replaced
|
||||||
|
assert host.active is (prior_posture == "active")
|
||||||
|
|
||||||
|
|
||||||
def test_first_install_waits_for_complete_cid_and_healthy_container(tmp_path: Path) -> None:
|
def test_first_install_waits_for_complete_cid_and_healthy_container(tmp_path: Path) -> None:
|
||||||
bundle, release, image = release_bundle(tmp_path, "2")
|
bundle, release, image = release_bundle(tmp_path, "2")
|
||||||
paths = install_paths(tmp_path)
|
paths = install_paths(tmp_path)
|
||||||
|
|
@ -1135,7 +1568,7 @@ def test_upgrade_stops_verified_prior_release_before_unit_replacement(tmp_path:
|
||||||
execute_restart=True,
|
execute_restart=True,
|
||||||
)
|
)
|
||||||
start = len(host.calls)
|
start = len(host.calls)
|
||||||
receipt = installer.install_release(
|
result = installer.install_release(
|
||||||
second_bundle,
|
second_bundle,
|
||||||
paths=paths,
|
paths=paths,
|
||||||
source_binding=source_binding(),
|
source_binding=source_binding(),
|
||||||
|
|
@ -1143,14 +1576,101 @@ def test_upgrade_stops_verified_prior_release_before_unit_replacement(tmp_path:
|
||||||
sleeper=lambda _seconds: None,
|
sleeper=lambda _seconds: None,
|
||||||
execute_restart=True,
|
execute_restart=True,
|
||||||
)
|
)
|
||||||
|
receipt = json.loads(paths.receipt.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
actions = [call[1] for call in host.calls[start:] if call[0] == installer.SYSTEMCTL]
|
actions = [call[1] for call in host.calls[start:] if call[0] == installer.SYSTEMCTL]
|
||||||
assert actions.index("stop") < actions.index("daemon-reload") < actions.index("restart")
|
assert actions.index("stop") < actions.index("daemon-reload") < actions.index("restart")
|
||||||
assert receipt["release_sha256"] == second_release["release_sha256"]
|
assert result["release_sha256"] == second_release["release_sha256"]
|
||||||
|
assert result["outcome"] == "replaced"
|
||||||
|
assert result["transition"] == {
|
||||||
|
"outcome": "replaced",
|
||||||
|
"prior_posture": "active",
|
||||||
|
"prior_n_restarts": 0,
|
||||||
|
"final_posture": "active",
|
||||||
|
"service_action": "restart",
|
||||||
|
"managed_files_replaced": True,
|
||||||
|
}
|
||||||
|
assert receipt["rollback"]["previous_service_posture"] == "active"
|
||||||
assert host.current_release == second_release
|
assert host.current_release == second_release
|
||||||
assert host.stop_calls == 1
|
assert host.stop_calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_inactive_prior_release_is_replaced_without_claiming_active_posture(tmp_path: Path) -> None:
|
||||||
|
first_bundle, first_release, first_image = release_bundle(tmp_path, "2")
|
||||||
|
second_bundle, second_release, second_image = release_bundle(tmp_path, "4")
|
||||||
|
paths = install_paths(tmp_path)
|
||||||
|
host = FakeHost(paths)
|
||||||
|
host.add_release(first_release, first_image)
|
||||||
|
host.add_release(second_release, second_image)
|
||||||
|
installer.install_release(
|
||||||
|
first_bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
host._stop()
|
||||||
|
stop_calls = host.stop_calls
|
||||||
|
|
||||||
|
result = installer.install_release(
|
||||||
|
second_bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
receipt = json.loads(paths.receipt.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
assert result["outcome"] == "replaced"
|
||||||
|
assert result["transition"]["prior_posture"] == "inactive"
|
||||||
|
assert result["transition"]["prior_n_restarts"] == 0
|
||||||
|
assert receipt["rollback"]["previous_release_present"] is True
|
||||||
|
assert receipt["rollback"]["previous_service_active"] is False
|
||||||
|
assert receipt["rollback"]["previous_service_posture"] == "inactive"
|
||||||
|
assert host.stop_calls == stop_calls
|
||||||
|
assert host.active
|
||||||
|
|
||||||
|
|
||||||
|
def test_active_prior_verification_failure_reports_unchanged_recovery_posture(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
first_bundle, first_release, first_image = release_bundle(tmp_path, "2")
|
||||||
|
second_bundle, second_release, second_image = release_bundle(tmp_path, "4")
|
||||||
|
paths = install_paths(tmp_path)
|
||||||
|
host = FakeHost(paths)
|
||||||
|
host.add_release(first_release, first_image)
|
||||||
|
host.add_release(second_release, second_image)
|
||||||
|
installer.install_release(
|
||||||
|
first_bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
process_root = paths.proc_root / str(host.main_pid)
|
||||||
|
(process_root / "environ").write_bytes(b"PATH=/unexpected\0")
|
||||||
|
stop_calls = host.stop_calls
|
||||||
|
|
||||||
|
with pytest.raises(installer.VerificationError) as caught:
|
||||||
|
installer.install_release(
|
||||||
|
second_bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert caught.value.rollback_executed is False
|
||||||
|
assert caught.value.restored_posture == "active"
|
||||||
|
assert caught.value.prior_posture_preserved is True
|
||||||
|
assert host.stop_calls == stop_calls
|
||||||
|
assert host.active
|
||||||
|
|
||||||
|
|
||||||
def test_stale_loaded_unit_is_rejected_before_trusting_old_exec_stop(tmp_path: Path) -> None:
|
def test_stale_loaded_unit_is_rejected_before_trusting_old_exec_stop(tmp_path: Path) -> None:
|
||||||
first_bundle, first_release, first_image = release_bundle(tmp_path, "2")
|
first_bundle, first_release, first_image = release_bundle(tmp_path, "2")
|
||||||
second_bundle, second_release, second_image = release_bundle(tmp_path, "4")
|
second_bundle, second_release, second_image = release_bundle(tmp_path, "4")
|
||||||
|
|
@ -1327,6 +1847,45 @@ def test_transitional_prior_service_posture_is_rejected_before_mutation(tmp_path
|
||||||
assert host.restart_calls == 1
|
assert host.restart_calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_prior_service_posture_is_rejected_before_mutation(tmp_path: Path) -> None:
|
||||||
|
first_bundle, first_release, first_image = release_bundle(tmp_path, "2")
|
||||||
|
second_bundle, second_release, second_image = release_bundle(tmp_path, "4")
|
||||||
|
paths = install_paths(tmp_path)
|
||||||
|
host = FakeHost(paths)
|
||||||
|
host.add_release(first_release, first_image)
|
||||||
|
host.add_release(second_release, second_image)
|
||||||
|
installer.install_release(
|
||||||
|
first_bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
host.property_overrides.update(
|
||||||
|
{
|
||||||
|
"ActiveState": "failed",
|
||||||
|
"SubState": "failed",
|
||||||
|
"MainPID": "0",
|
||||||
|
"InvocationID": "",
|
||||||
|
"ControlGroup": "",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(installer.InstallError, match="transitional or failed"):
|
||||||
|
installer.install_release(
|
||||||
|
second_bundle,
|
||||||
|
paths=paths,
|
||||||
|
source_binding=source_binding(),
|
||||||
|
runner=host,
|
||||||
|
sleeper=lambda _seconds: None,
|
||||||
|
execute_restart=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert host.stop_calls == 0
|
||||||
|
assert host.restart_calls == 1
|
||||||
|
|
||||||
|
|
||||||
def test_changing_prior_service_posture_fails_two_readback_gate(tmp_path: Path) -> None:
|
def test_changing_prior_service_posture_fails_two_readback_gate(tmp_path: Path) -> None:
|
||||||
first_bundle, first_release, first_image = release_bundle(tmp_path, "2")
|
first_bundle, first_release, first_image = release_bundle(tmp_path, "2")
|
||||||
second_bundle, second_release, second_image = release_bundle(tmp_path, "4")
|
second_bundle, second_release, second_image = release_bundle(tmp_path, "4")
|
||||||
|
|
@ -1366,7 +1925,7 @@ def test_wrong_local_image_is_rejected_without_service_restart(tmp_path: Path) -
|
||||||
host = FakeHost(paths)
|
host = FakeHost(paths)
|
||||||
host.add_release(release, image)
|
host.add_release(release, image)
|
||||||
|
|
||||||
with pytest.raises(installer.InstallError, match=r"finalized (release|inspection)"):
|
with pytest.raises(installer.VerificationError, match=r"finalized (release|inspection)") as caught:
|
||||||
installer.install_release(
|
installer.install_release(
|
||||||
bundle,
|
bundle,
|
||||||
paths=paths,
|
paths=paths,
|
||||||
|
|
@ -1376,6 +1935,9 @@ def test_wrong_local_image_is_rejected_without_service_restart(tmp_path: Path) -
|
||||||
execute_restart=True,
|
execute_restart=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
assert caught.value.rollback_executed is False
|
||||||
|
assert caught.value.restored_posture == "absent"
|
||||||
|
assert caught.value.prior_posture_preserved is True
|
||||||
assert host.restart_calls == 0
|
assert host.restart_calls == 0
|
||||||
assert not paths.unit.exists()
|
assert not paths.unit.exists()
|
||||||
assert not paths.state_dir.exists()
|
assert not paths.state_dir.exists()
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue