feat(#60): 一键部署编排与回滚引擎
纯标准库实现，对齐 PRD 5.6「⑥ 部署底座」（不真执行 k8s/helm，模拟编排）： - deploy_plan.py：DeployStep（4 类：pre_check/deploy/health_check/post_check） + DeployPlan（顺序约束自动校验，default_helm_release 默认 4 步计划）+ DeployOrchestrator（按序执行，必需步骤失败即中止并标记 needs_rollback， deploy/health_check 为回滚检查点，pre_check 失败无需回滚）+ HealthCheckContract（对齐 deploy/k8s/healthz 99.8% 目标）。 - rollback.py：RollbackPoint（部署前快照，values_hash SHA1 检测漂移）+ RollbackManager（快照栈 + rollback_to_latest/rollback(n)，失败压回栈顶 保持一致性，FIFO 淘汰，支持回滚后健康检查）+ RollbackResult（4 状态）。 - tests：30 用例覆盖计划构造/顺序约束、全成功/各类失败、回滚检查点判定、 action 异常、dry_run、空计划、快照压栈/FIFO、回滚成功/失败/多版本/空栈/ 异常、健康检查契约、values 漂移、报告序列化。 - _sanity_check.py：冒烟验证快照→部署成功→部署失败→回滚全流程。
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""测试引导：把 ``deploy/orchestration`` 挂到 sys.path，使 deploy_plan/rollback
|
||||
可被 ``from deploy_plan import ...`` 导入（orchestration 包内相对导入需要）。
|
||||
|
||||
与 core 模块测试引导同款模式。
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
|
||||
PKG_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PKG_DIR not in sys.path:
|
||||
sys.path.insert(0, PKG_DIR)
|
||||
@@ -0,0 +1,200 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""部署编排引擎测试（Issue #60）。
|
||||
|
||||
覆盖：
|
||||
1. 默认 Helm 计划构造（4 步：pre_check/deploy/health_check/post_check）；
|
||||
2. 步骤顺序约束校验（违例 → DeployError）；
|
||||
3. 全成功执行（succeeded + 无需回滚）；
|
||||
4. deploy 步骤失败（needs_rollback + failed_step）；
|
||||
5. pre_check 失败（needs_rollback=False，非回滚检查点）；
|
||||
6. 非必需 post_check 失败（继续编排，succeeded）；
|
||||
7. action 异常（视为失败）；
|
||||
8. dry_run（全部 SKIPPED）；
|
||||
9. 空计划（succeeded=False）；
|
||||
10. HealthCheckContract 校验；
|
||||
11. 报告序列化。
|
||||
"""
|
||||
import unittest
|
||||
|
||||
import _bootstrap # noqa: F401 (sys.path 挂载)
|
||||
|
||||
from deploy_plan import (
|
||||
DEFAULT_AVAILABILITY_TARGET,
|
||||
DeployError,
|
||||
DeployOrchestrator,
|
||||
DeployOutcome,
|
||||
DeployPlan,
|
||||
DeployStep,
|
||||
DeployStepKind,
|
||||
DeployStatus,
|
||||
HealthCheckContract,
|
||||
)
|
||||
|
||||
|
||||
def _action(ok: bool, detail: str = ""):
|
||||
"""构造固定返回值的 action。"""
|
||||
def _fn(ctx):
|
||||
return ok, (detail or ("成功" if ok else "失败"))
|
||||
return _fn
|
||||
|
||||
|
||||
def _raising_action(exc_type=RuntimeError):
|
||||
def _fn(ctx):
|
||||
raise exc_type("boom")
|
||||
return _fn
|
||||
|
||||
|
||||
class TestDeployPlanModel(unittest.TestCase):
|
||||
"""部署计划模型与顺序约束。"""
|
||||
|
||||
def test_default_helm_release_has_four_steps(self):
|
||||
plan = DeployPlan.default_helm_release()
|
||||
kinds = [s.kind for s in plan.steps]
|
||||
self.assertEqual(kinds, [
|
||||
DeployStepKind.PRE_CHECK,
|
||||
DeployStepKind.DEPLOY,
|
||||
DeployStepKind.HEALTH_CHECK,
|
||||
DeployStepKind.POST_CHECK,
|
||||
])
|
||||
# post_check 默认非必需
|
||||
self.assertFalse(plan.steps[-1].required)
|
||||
|
||||
def test_step_order_violation_rejected(self):
|
||||
# health_check 排在 deploy 前 → 违例
|
||||
with self.assertRaises(DeployError):
|
||||
DeployPlan(
|
||||
name="bad",
|
||||
steps=[
|
||||
DeployStep(DeployStepKind.HEALTH_CHECK, "hc"),
|
||||
DeployStep(DeployStepKind.DEPLOY, "d"),
|
||||
])
|
||||
|
||||
def test_empty_step_name_rejected(self):
|
||||
with self.assertRaises(DeployError):
|
||||
DeployStep(DeployStepKind.DEPLOY, "")
|
||||
|
||||
def test_plan_name_required(self):
|
||||
with self.assertRaises(DeployError):
|
||||
DeployPlan(name="")
|
||||
|
||||
|
||||
class TestOrchestratorSuccess(unittest.TestCase):
|
||||
"""全成功编排。"""
|
||||
|
||||
def test_all_steps_succeed(self):
|
||||
plan = DeployPlan.default_helm_release()
|
||||
report = DeployOrchestrator(plan).execute()
|
||||
self.assertTrue(report.succeeded)
|
||||
self.assertFalse(report.outcome.needs_rollback)
|
||||
self.assertEqual(len(report.results), 4)
|
||||
self.assertEqual(
|
||||
[r.status for r in report.results],
|
||||
[DeployStatus.SUCCESS] * 4)
|
||||
# 日志含 START/RUN/OK
|
||||
self.assertTrue(any("[START]" in ln for ln in report.log_lines))
|
||||
self.assertTrue(any("[OK]" in ln for ln in report.log_lines))
|
||||
|
||||
def test_context_populated(self):
|
||||
plan = DeployPlan.default_helm_release()
|
||||
report = DeployOrchestrator(plan).execute()
|
||||
self.assertEqual(report.context["release"], "iaop")
|
||||
self.assertEqual(report.context["namespace"], "iaop")
|
||||
self.assertIn("health", report.context)
|
||||
|
||||
|
||||
class TestOrchestratorFailures(unittest.TestCase):
|
||||
"""失败场景 + 回滚标记。"""
|
||||
|
||||
def test_deploy_failure_triggers_rollback(self):
|
||||
plan = DeployPlan.default_helm_release(
|
||||
step_actions={DeployStepKind.DEPLOY: _action(False, "helm 失败")})
|
||||
report = DeployOrchestrator(plan).execute()
|
||||
self.assertFalse(report.succeeded)
|
||||
self.assertTrue(report.outcome.needs_rollback) # deploy 是回滚检查点
|
||||
self.assertIsNotNone(report.outcome.failed_step)
|
||||
# deploy 之后的 health_check/post_check 未执行
|
||||
executed_kinds = [r.step.kind for r in report.results]
|
||||
self.assertNotIn(DeployStepKind.HEALTH_CHECK, executed_kinds)
|
||||
self.assertNotIn(DeployStepKind.POST_CHECK, executed_kinds)
|
||||
|
||||
def test_pre_check_failure_no_rollback(self):
|
||||
plan = DeployPlan.default_helm_release(
|
||||
step_actions={DeployStepKind.PRE_CHECK: _action(False, "集群不可达")})
|
||||
report = DeployOrchestrator(plan).execute()
|
||||
self.assertFalse(report.succeeded)
|
||||
# pre_check 失败：尚未部署，无需回滚
|
||||
self.assertFalse(report.outcome.needs_rollback)
|
||||
self.assertIn("集群不可达", report.outcome.reason)
|
||||
|
||||
def test_non_required_post_check_failure_continues(self):
|
||||
plan = DeployPlan.default_helm_release(
|
||||
step_actions={DeployStepKind.POST_CHECK: _action(False, "烟测失败")})
|
||||
report = DeployOrchestrator(plan).execute()
|
||||
# post_check 非必需，失败仍判成功（核心步骤都过了）
|
||||
self.assertTrue(report.succeeded)
|
||||
self.assertFalse(report.outcome.needs_rollback)
|
||||
# 但有 WARN 日志
|
||||
self.assertTrue(any("[WARN]" in ln for ln in report.log_lines))
|
||||
|
||||
def test_action_exception_treated_as_failure(self):
|
||||
plan = DeployPlan.default_helm_release(
|
||||
step_actions={DeployStepKind.DEPLOY: _raising_action()})
|
||||
report = DeployOrchestrator(plan).execute()
|
||||
self.assertFalse(report.succeeded)
|
||||
self.assertTrue(report.outcome.needs_rollback)
|
||||
self.assertIn("action 异常", report.outcome.reason)
|
||||
|
||||
def test_health_check_failure_triggers_rollback(self):
|
||||
plan = DeployPlan.default_helm_release(
|
||||
step_actions={DeployStepKind.HEALTH_CHECK: _action(False, "/health 不可用")})
|
||||
report = DeployOrchestrator(plan).execute()
|
||||
self.assertFalse(report.succeeded)
|
||||
self.assertTrue(report.outcome.needs_rollback)
|
||||
|
||||
|
||||
class TestDryRunAndEmpty(unittest.TestCase):
|
||||
"""dry_run + 空计划。"""
|
||||
|
||||
def test_dry_run_skips_all(self):
|
||||
plan = DeployPlan.default_helm_release()
|
||||
report = DeployOrchestrator(plan, dry_run=True).execute()
|
||||
self.assertTrue(report.succeeded)
|
||||
self.assertEqual(
|
||||
[r.status for r in report.results],
|
||||
[DeployStatus.SKIPPED] * 4)
|
||||
|
||||
def test_empty_plan_fails(self):
|
||||
plan = DeployPlan(name="empty", steps=[])
|
||||
report = DeployOrchestrator(plan).execute()
|
||||
self.assertFalse(report.succeeded)
|
||||
self.assertIn("无步骤", report.outcome.reason)
|
||||
|
||||
|
||||
class TestHealthContractAndReport(unittest.TestCase):
|
||||
"""HealthCheckContract 校验 + 报告序列化。"""
|
||||
|
||||
def test_health_contract_defaults(self):
|
||||
h = HealthCheckContract()
|
||||
self.assertEqual(h.endpoint, "http://iaop:8000/health")
|
||||
self.assertEqual(h.retries, 3)
|
||||
self.assertAlmostEqual(h.availability_target, DEFAULT_AVAILABILITY_TARGET)
|
||||
|
||||
def test_health_contract_validation(self):
|
||||
with self.assertRaises(DeployError):
|
||||
HealthCheckContract(timeout_s=0)
|
||||
with self.assertRaises(DeployError):
|
||||
HealthCheckContract(retries=0)
|
||||
with self.assertRaises(DeployError):
|
||||
HealthCheckContract(availability_target=1.5)
|
||||
|
||||
def test_report_to_dict(self):
|
||||
plan = DeployPlan.default_helm_release()
|
||||
report = DeployOrchestrator(plan).execute()
|
||||
d = report.to_dict()
|
||||
self.assertTrue(d["succeeded"])
|
||||
self.assertEqual(len(d["steps"]), 4)
|
||||
self.assertEqual(d["steps"][0]["kind"], "pre_check")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,185 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""部署回滚引擎测试（Issue #60）。
|
||||
|
||||
覆盖：
|
||||
1. 快照压栈 + latest 查询；
|
||||
2. FIFO 淘汰（超 max_snapshots）；
|
||||
3. rollback_to_latest 成功（弹出栈顶 + 回滚目标版本）；
|
||||
4. rollback(n) 多版本回溯；
|
||||
5. 空栈回滚（NO_TARGET）；
|
||||
6. restore 动作失败（FAILED + 压回栈顶）；
|
||||
7. restore 异常（FAILED）；
|
||||
8. post_rollback_health 失败（FAILED + 压回）；
|
||||
9. n<1 跳过；
|
||||
10. values_hash 漂移检测；
|
||||
11. 报告序列化。
|
||||
"""
|
||||
import unittest
|
||||
|
||||
import _bootstrap # noqa: F401 (sys.path 挂载)
|
||||
|
||||
from deploy_plan import DeployPlan
|
||||
from rollback import (
|
||||
DeployError,
|
||||
RollbackManager,
|
||||
RollbackPoint,
|
||||
RollbackStatus,
|
||||
)
|
||||
|
||||
|
||||
def _restore(ok: bool, detail: str = ""):
|
||||
def _fn(snapshot, ctx):
|
||||
return ok, (detail or ("回滚成功" if ok else "回滚失败"))
|
||||
return _fn
|
||||
|
||||
|
||||
def _raising_restore():
|
||||
def _fn(snapshot, ctx):
|
||||
raise RuntimeError("restore boom")
|
||||
return _fn
|
||||
|
||||
|
||||
def _make_plan(version="v1.0.0", release="iaop"):
|
||||
return DeployPlan.default_helm_release(release=release, version=version)
|
||||
|
||||
|
||||
class TestSnapshotStack(unittest.TestCase):
|
||||
"""快照压栈与查询。"""
|
||||
|
||||
def test_snapshot_push_and_latest(self):
|
||||
mgr = RollbackManager()
|
||||
# snapshot 即压栈（部署前快照），latest 即栈顶
|
||||
mgr.snapshot(_make_plan("v1.0.0"), values={"k": 1}, reason="第一次")
|
||||
self.assertEqual(mgr.latest().version, "v1.0.0")
|
||||
self.assertEqual(len(mgr.snapshots), 1)
|
||||
mgr.snapshot(_make_plan("v1.1.0"), values={"k": 2})
|
||||
self.assertEqual(mgr.latest().version, "v1.1.0")
|
||||
self.assertEqual(len(mgr.snapshots), 2)
|
||||
# 空栈时 latest 返回 None
|
||||
self.assertIsNotNone(mgr.latest())
|
||||
mgr2 = RollbackManager()
|
||||
self.assertIsNone(mgr2.latest())
|
||||
|
||||
def test_fifo_eviction(self):
|
||||
mgr = RollbackManager(max_snapshots=2)
|
||||
mgr.snapshot(_make_plan("v1.0.0"))
|
||||
mgr.snapshot(_make_plan("v1.1.0"))
|
||||
mgr.snapshot(_make_plan("v1.2.0"))
|
||||
# 上限 2，淘汰最旧
|
||||
self.assertEqual(len(mgr.snapshots), 2)
|
||||
self.assertEqual([s.version for s in mgr.snapshots], ["v1.1.0", "v1.2.0"])
|
||||
|
||||
def test_max_snapshots_validation(self):
|
||||
with self.assertRaises(DeployError):
|
||||
RollbackManager(max_snapshots=0)
|
||||
|
||||
def test_values_hash_changes_with_config(self):
|
||||
mgr = RollbackManager()
|
||||
p1 = mgr.snapshot(_make_plan(), values={"backend": "gpu"})
|
||||
p2 = mgr.snapshot(_make_plan(), values={"backend": "npu"})
|
||||
self.assertNotEqual(p1.values_hash, p2.values_hash)
|
||||
|
||||
|
||||
class TestRollbackSuccess(unittest.TestCase):
|
||||
"""回滚成功场景。"""
|
||||
|
||||
def test_rollback_to_latest_success(self):
|
||||
mgr = RollbackManager(restore_action=_restore(True))
|
||||
mgr.snapshot(_make_plan("v1.0.0"))
|
||||
mgr.snapshot(_make_plan("v1.1.0"))
|
||||
result = mgr.rollback_to_latest(reason="部署 v1.2.0 失败")
|
||||
self.assertTrue(result.succeeded)
|
||||
self.assertEqual(result.status, RollbackStatus.SUCCESS)
|
||||
self.assertEqual(result.target.version, "v1.1.0")
|
||||
# 弹出栈顶后栈剩 1 个
|
||||
self.assertEqual(len(mgr.snapshots), 1)
|
||||
self.assertEqual(mgr.latest().version, "v1.0.0")
|
||||
self.assertIn("v1.2.0 失败", result.reason)
|
||||
|
||||
def test_rollback_n_versions(self):
|
||||
mgr = RollbackManager(restore_action=_restore(True))
|
||||
for v in ["v1.0.0", "v1.1.0", "v1.2.0"]:
|
||||
mgr.snapshot(_make_plan(v))
|
||||
# 回滚 2 个版本 → 目标 v1.1.0（弹出 v1.2.0 和 v1.1.0）
|
||||
result = mgr.rollback(n=2)
|
||||
self.assertTrue(result.succeeded)
|
||||
self.assertEqual(result.target.version, "v1.1.0")
|
||||
self.assertEqual(len(mgr.snapshots), 1)
|
||||
|
||||
|
||||
class TestRollbackFailures(unittest.TestCase):
|
||||
"""回滚失败与边界。"""
|
||||
|
||||
def test_empty_stack_no_target(self):
|
||||
mgr = RollbackManager()
|
||||
result = mgr.rollback_to_latest()
|
||||
self.assertEqual(result.status, RollbackStatus.NO_TARGET)
|
||||
self.assertFalse(result.succeeded)
|
||||
|
||||
def test_insufficient_stack_for_n(self):
|
||||
mgr = RollbackManager(restore_action=_restore(True))
|
||||
mgr.snapshot(_make_plan("v1.0.0"))
|
||||
# 栈仅 1 个，回滚 2 个 → NO_TARGET
|
||||
result = mgr.rollback(n=2)
|
||||
self.assertEqual(result.status, RollbackStatus.NO_TARGET)
|
||||
|
||||
def test_restore_failure_pushes_target_back(self):
|
||||
mgr = RollbackManager(restore_action=_restore(False, "helm rollback 失败"))
|
||||
mgr.snapshot(_make_plan("v1.0.0"))
|
||||
mgr.snapshot(_make_plan("v1.1.0"))
|
||||
result = mgr.rollback_to_latest()
|
||||
self.assertEqual(result.status, RollbackStatus.FAILED)
|
||||
# 失败时目标压回栈顶，栈仍 2 个
|
||||
self.assertEqual(len(mgr.snapshots), 2)
|
||||
self.assertEqual(mgr.latest().version, "v1.1.0")
|
||||
|
||||
def test_restore_exception_failure(self):
|
||||
mgr = RollbackManager(restore_action=_raising_restore())
|
||||
mgr.snapshot(_make_plan("v1.0.0"))
|
||||
result = mgr.rollback_to_latest()
|
||||
self.assertEqual(result.status, RollbackStatus.FAILED)
|
||||
self.assertIn("异常", result.detail)
|
||||
|
||||
def test_post_rollback_health_failure(self):
|
||||
# restore 成功但健康检查失败（注入失败的健康检查）
|
||||
def _restore_then_health_fail(snapshot, ctx):
|
||||
return True, "回滚动作成功"
|
||||
mgr = RollbackManager(
|
||||
restore_action=_restore_then_health_fail, post_rollback_health=True)
|
||||
# 覆盖 _health_check 为失败
|
||||
mgr._health_check = lambda target: (False, "/health 仍不可用")
|
||||
mgr.snapshot(_make_plan("v1.0.0"))
|
||||
result = mgr.rollback_to_latest()
|
||||
self.assertEqual(result.status, RollbackStatus.FAILED)
|
||||
self.assertIn("健康检查", result.detail)
|
||||
# 压回栈顶
|
||||
self.assertEqual(len(mgr.snapshots), 1)
|
||||
|
||||
def test_n_less_than_one_skipped(self):
|
||||
mgr = RollbackManager()
|
||||
result = mgr.rollback(n=0)
|
||||
self.assertEqual(result.status, RollbackStatus.SKIPPED)
|
||||
|
||||
|
||||
class TestResultExport(unittest.TestCase):
|
||||
"""结果序列化。"""
|
||||
|
||||
def test_result_to_dict(self):
|
||||
mgr = RollbackManager(restore_action=_restore(True))
|
||||
mgr.snapshot(_make_plan("v1.0.0"))
|
||||
result = mgr.rollback_to_latest()
|
||||
d = result.to_dict()
|
||||
self.assertEqual(d["status"], "success")
|
||||
self.assertTrue(d["succeeded"])
|
||||
self.assertEqual(d["target"]["version"], "v1.0.0")
|
||||
|
||||
def test_point_to_dict(self):
|
||||
p = RollbackPoint(release="r", namespace="ns", version="v1",
|
||||
chart="c", values_hash="abc123", reason="x")
|
||||
d = p.to_dict()
|
||||
self.assertEqual(d["version"], "v1")
|
||||
self.assertEqual(d["values_hash"], "abc123")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user