106 lines
4.8 KiB
Python
106 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
"""Failure tests for Phase-1.0AO worker supervision."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
import sys
|
|
import unittest
|
|
|
|
|
|
PARSER = argparse.ArgumentParser()
|
|
PARSER.add_argument("--root", type=Path, required=True)
|
|
ROOT = PARSER.parse_args().root
|
|
sys.path.insert(0, str(ROOT / "tools"))
|
|
|
|
from phase10ao_worker_supervisor_model import * # noqa: E402,F403
|
|
|
|
|
|
PLAN = WorkerPlan(101, 202, 4096)
|
|
|
|
|
|
class WorkerSupervisorTests(unittest.TestCase):
|
|
def test_exact_success_reaps_worker_and_retains_child(self) -> None:
|
|
result = WorkerResult(101, SUCCESS, 4096, 0)
|
|
events = (FakeSupervisorEvent(CREATE_WORKER, OK),
|
|
FakeSupervisorEvent(VERIFY_WORKER, OK),
|
|
FakeSupervisorEvent(START_COPY, OK),
|
|
FakeSupervisorEvent(RECEIVE_RESULT, OK, result),
|
|
FakeSupervisorEvent(REAP_WORKER, OK))
|
|
outcome = run_supervisor(PLAN, FakeSupervisorFacade(events))
|
|
self.assertTrue(outcome.success and outcome.service_alive and outcome.child_alive)
|
|
self.assertFalse(outcome.worker_alive or outcome.automatic_restart)
|
|
self.assertFalse(outcome.target_action_performed)
|
|
|
|
def test_deadline_terminates_worker_and_child(self) -> None:
|
|
events = (FakeSupervisorEvent(CREATE_WORKER, OK),
|
|
FakeSupervisorEvent(VERIFY_WORKER, OK),
|
|
FakeSupervisorEvent(START_COPY, OK),
|
|
FakeSupervisorEvent(DEADLINE, OK),
|
|
FakeSupervisorEvent(TERMINATE_WORKER, OK),
|
|
FakeSupervisorEvent(REAP_WORKER, OK),
|
|
FakeSupervisorEvent(TERMINATE_CHILD, OK),
|
|
FakeSupervisorEvent(REAP_CHILD, OK))
|
|
outcome = run_supervisor(PLAN, FakeSupervisorFacade(events), True)
|
|
self.assertEqual(outcome.classification, "OFFLINE_DEADLINE_CONTAINED")
|
|
self.assertTrue(outcome.service_alive)
|
|
self.assertFalse(outcome.worker_alive or outcome.child_alive)
|
|
|
|
def test_wrong_identity_partial_and_restore_failure_are_contained(self) -> None:
|
|
results = (WorkerResult(999, SUCCESS, 4096, 0),
|
|
WorkerResult(101, COPY_ERROR, 7, 0),
|
|
WorkerResult(101, RESTORE_ERROR, 4096, 1))
|
|
for result in results:
|
|
events = (FakeSupervisorEvent(CREATE_WORKER, OK),
|
|
FakeSupervisorEvent(VERIFY_WORKER, OK),
|
|
FakeSupervisorEvent(START_COPY, OK),
|
|
FakeSupervisorEvent(RECEIVE_RESULT, OK, result),
|
|
FakeSupervisorEvent(TERMINATE_WORKER, OK),
|
|
FakeSupervisorEvent(REAP_WORKER, OK),
|
|
FakeSupervisorEvent(TERMINATE_CHILD, OK),
|
|
FakeSupervisorEvent(REAP_CHILD, OK))
|
|
with self.subTest(result=result):
|
|
outcome = run_supervisor(PLAN, FakeSupervisorFacade(events))
|
|
self.assertFalse(outcome.success or outcome.child_alive)
|
|
|
|
def test_start_ambiguity_contains_both_processes(self) -> None:
|
|
events = (FakeSupervisorEvent(CREATE_WORKER, OK),
|
|
FakeSupervisorEvent(VERIFY_WORKER, OK),
|
|
FakeSupervisorEvent(START_COPY, ERROR),
|
|
FakeSupervisorEvent(TERMINATE_WORKER, OK),
|
|
FakeSupervisorEvent(REAP_WORKER, OK),
|
|
FakeSupervisorEvent(TERMINATE_CHILD, OK),
|
|
FakeSupervisorEvent(REAP_CHILD, OK))
|
|
outcome = run_supervisor(PLAN, FakeSupervisorFacade(events))
|
|
self.assertFalse(outcome.child_alive)
|
|
|
|
def test_pre_start_failure_does_not_kill_untouched_child(self) -> None:
|
|
events = (FakeSupervisorEvent(CREATE_WORKER, OK),
|
|
FakeSupervisorEvent(VERIFY_WORKER, ERROR),
|
|
FakeSupervisorEvent(TERMINATE_WORKER, OK),
|
|
FakeSupervisorEvent(REAP_WORKER, OK))
|
|
outcome = run_supervisor(PLAN, FakeSupervisorFacade(events))
|
|
self.assertTrue(outcome.child_alive)
|
|
|
|
def test_any_terminal_failure_is_hard(self) -> None:
|
|
events = (FakeSupervisorEvent(CREATE_WORKER, OK),
|
|
FakeSupervisorEvent(VERIFY_WORKER, ERROR),
|
|
FakeSupervisorEvent(TERMINATE_WORKER, ERROR))
|
|
with self.assertRaises(SupervisorError):
|
|
run_supervisor(PLAN, FakeSupervisorFacade(events))
|
|
|
|
def test_bounds_and_unused_events_are_rejected(self) -> None:
|
|
for values in ((0, 2, 1), (1, 1, 1), (1, 2, MAX_COPY_SIZE + 1)):
|
|
with self.subTest(values=values), self.assertRaises(SupervisorError):
|
|
WorkerPlan(*values)
|
|
events = (FakeSupervisorEvent(CREATE_WORKER, ERROR),
|
|
FakeSupervisorEvent(REAP_WORKER, OK))
|
|
with self.assertRaises(SupervisorError):
|
|
run_supervisor(PLAN, FakeSupervisorFacade(events))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(argv=[__file__])
|