331 lines
17 KiB
Python
331 lines
17 KiB
Python
"""有限恢复的隔离行为回归;所有来源/构建/模型替身,计划只写临时目录。"""
|
||
import contextlib
|
||
import datetime as dt
|
||
import fcntl
|
||
import json
|
||
from pathlib import Path
|
||
import socket
|
||
import sys
|
||
import tempfile
|
||
import types
|
||
import unittest
|
||
from unittest.mock import Mock, patch
|
||
|
||
import config
|
||
import plan
|
||
import plan_recovery as recovery
|
||
import plan_versions as pv
|
||
import run
|
||
|
||
GENERATE = plan._generate
|
||
|
||
NOW = dt.datetime.fromisoformat('2026-11-05T10:05:00+08:00')
|
||
DAY = '2026-11-04'
|
||
|
||
|
||
class RecoveryTest(unittest.TestCase):
|
||
def setUp(self):
|
||
self.stack = contextlib.ExitStack()
|
||
self.addCleanup(self.stack.close)
|
||
self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))
|
||
self.stack.enter_context(patch.object(config, 'PLAN_SNAPSHOT_DIR', str(self.root)))
|
||
self.stack.enter_context(patch.object(config, 'PLAN_RECOVERY_MAX_ATTEMPTS', 3))
|
||
self.stack.enter_context(patch.object(config, 'PLAN_RECOVERY_COOLDOWN_SECONDS', 900))
|
||
self.stack.enter_context(patch.object(socket.socket, 'connect', side_effect=AssertionError('禁止联网')))
|
||
self.ready = self.stack.enter_context(patch.object(recovery.build_readiness, 'check', return_value=DAY))
|
||
self.build = self.stack.enter_context(patch.object(run, '_cmd_build', return_value=DAY))
|
||
self.generate = self.stack.enter_context(patch.object(plan, '_generate', side_effect=self.publish))
|
||
self.pool = Mock(side_effect=AssertionError('恢复不入池'))
|
||
self.model = Mock(side_effect=AssertionError('恢复不额外发模型任务'))
|
||
self.stack.enter_context(patch.dict(sys.modules, {
|
||
'pool': types.SimpleNamespace(push=self.pool),
|
||
'user_research': types.SimpleNamespace(sync=self.model),
|
||
}))
|
||
|
||
def publish(self, day=DAY, *args, cutoff=NOW, **kwargs):
|
||
if kwargs.get('before_publish'):
|
||
kwargs['before_publish'](day)
|
||
return pv.publish(str(self.root), {'date': day, 'main': [], 'observe': [],
|
||
'plan_version': 'isolated'}, {'records': {}, 'market': {}}, {},
|
||
cutoff_at=cutoff.isoformat(), reason='晨版生成')
|
||
|
||
def state(self, day='2026-11-05'):
|
||
return json.loads((self.root / 'recovery' / (day + '.json')).read_text())
|
||
|
||
def test_missing_inputs_wait_without_build_or_budget(self):
|
||
self.ready.side_effect = RuntimeError('资金热度日期 2026-11-03,预期 2026-11-04')
|
||
result = recovery.ensure_today(now=NOW)
|
||
self.assertEqual(result['status'], 'waiting_data')
|
||
self.assertIn('2026-11-03', result['reason'])
|
||
self.build.assert_not_called()
|
||
self.generate.assert_not_called()
|
||
self.assertFalse((self.root / 'recovery').exists())
|
||
|
||
def test_ready_builds_once_with_real_late_cutoff_and_no_pool_or_model(self):
|
||
old = self.publish('2026-11-03', cutoff=NOW - dt.timedelta(days=1))
|
||
old_path = self.root / 'plan_2026-11-03.json'
|
||
previous = old_path.read_bytes()
|
||
result = recovery.ensure_today(now=NOW)
|
||
self.assertEqual(result['status'], 'recovered')
|
||
self.build.assert_called_once_with('all', 'daily', None, None, None)
|
||
today = pv.load(self.root, DAY)
|
||
self.assertEqual(today['decision']['publication_kind'], 'recovery')
|
||
self.assertEqual(today['decision']['cutoff_at'], NOW.isoformat())
|
||
self.assertFalse(pv.morning_eligible(today))
|
||
self.assertEqual(old_path.read_bytes(), previous)
|
||
self.assertEqual(pv.load(self.root, '2026-11-03', morning=True), old)
|
||
self.assertEqual(self.state()['attempts'], 1)
|
||
self.assertEqual(self.state()['status'], 'recovered')
|
||
self.pool.assert_not_called()
|
||
self.model.assert_not_called()
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'already_ready')
|
||
self.build.assert_called_once()
|
||
|
||
def test_existing_recovery_does_not_rebuild_even_when_sources_unavailable(self):
|
||
self.publish()
|
||
self.ready.side_effect = AssertionError('已有有效版不重新取构建源')
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'already_ready')
|
||
self.build.assert_not_called()
|
||
self.ready.assert_not_called()
|
||
|
||
def test_explicit_date_never_automatically_rebuilds(self):
|
||
for day in ('2026-11-03', DAY):
|
||
self.assertEqual(recovery.ensure_today(day, now=NOW)['status'], 'explicit_date')
|
||
self.build.assert_not_called()
|
||
self.ready.assert_not_called()
|
||
|
||
def test_outside_window_and_weekend_do_not_rebuild(self):
|
||
for stamp in ('2026-11-05T07:19:59+08:00', '2026-11-05T15:10:01+08:00',
|
||
'2026-11-07T10:00:00+08:00'):
|
||
with self.subTest(stamp=stamp):
|
||
result = recovery.ensure_today(now=dt.datetime.fromisoformat(stamp))
|
||
self.assertEqual(result['status'], 'outside_window')
|
||
self.build.assert_not_called()
|
||
self.ready.assert_not_called()
|
||
|
||
def test_preexisting_legacy_or_wrong_decision_day_is_preserved(self):
|
||
path = self.root / ('plan_' + DAY + '.json')
|
||
for value in ({'date': DAY, 'main': []}, 'broken json'):
|
||
path.write_text(json.dumps(value) if isinstance(value, dict) else value)
|
||
before = path.read_bytes()
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'blocked_existing_snapshot')
|
||
self.assertEqual(path.read_bytes(), before)
|
||
path.unlink()
|
||
self.publish(cutoff=NOW - dt.timedelta(days=1))
|
||
before = path.read_bytes()
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'blocked_existing_snapshot')
|
||
self.assertEqual(path.read_bytes(), before)
|
||
self.build.assert_not_called()
|
||
|
||
def test_future_published_day_is_not_replaced_or_suppressed(self):
|
||
self.publish('2026-11-06')
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'blocked_existing_snapshot')
|
||
self.build.assert_not_called()
|
||
|
||
def test_lock_recheck_observes_other_success(self):
|
||
writer = pv.writer
|
||
@contextlib.contextmanager
|
||
def publish_before_lock(root):
|
||
with writer(root):
|
||
self.publish()
|
||
yield
|
||
with patch.object(pv, 'writer', side_effect=publish_before_lock):
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'already_ready')
|
||
self.build.assert_not_called()
|
||
|
||
def test_inputs_changed_while_waiting_lock_do_not_consume_attempt(self):
|
||
self.ready.side_effect = [DAY, RuntimeError('热度已失效')]
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'waiting_data')
|
||
self.build.assert_not_called()
|
||
self.assertFalse((self.root / 'recovery').exists())
|
||
|
||
def test_failed_build_persists_budget_and_obeys_cooldown_and_daily_cap(self):
|
||
self.build.side_effect = RuntimeError('构建失败')
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'failed')
|
||
self.assertEqual(self.state()['status'], 'failed')
|
||
self.assertEqual(recovery.ensure_today(now=NOW + dt.timedelta(minutes=14))['status'], 'cooldown')
|
||
self.assertEqual(self.build.call_count, 1)
|
||
self.assertEqual(recovery.ensure_today(now=NOW + dt.timedelta(minutes=15))['status'], 'failed')
|
||
self.assertEqual(recovery.ensure_today(now=NOW + dt.timedelta(minutes=30))['status'], 'failed')
|
||
self.assertEqual(recovery.ensure_today(now=NOW + dt.timedelta(minutes=45))['status'], 'attempts_exhausted')
|
||
self.assertEqual(self.build.call_count, 3)
|
||
self.assertEqual(self.state()['attempts'], 3)
|
||
self.generate.assert_not_called()
|
||
|
||
def test_persisted_running_attempt_survives_process_restart(self):
|
||
path = self.root / 'recovery' / '2026-11-05.json'
|
||
pv.atomic(path, {'schema_version': 1, 'decision_date': '2026-11-05', 'data_date': DAY,
|
||
'attempts': 2, 'status': 'building', 'last_attempt_at': NOW.isoformat()})
|
||
self.assertEqual(recovery.ensure_today(now=NOW + dt.timedelta(minutes=1))['status'], 'cooldown')
|
||
self.build.assert_not_called()
|
||
|
||
def test_day_switch_has_independent_bounded_budget(self):
|
||
self.build.side_effect = RuntimeError('构建失败')
|
||
for minutes in (0, 15, 30):
|
||
recovery.ensure_today(now=NOW + dt.timedelta(minutes=minutes))
|
||
self.ready.return_value = '2026-11-05'
|
||
tomorrow = NOW + dt.timedelta(days=1)
|
||
self.assertEqual(recovery.ensure_today(now=tomorrow)['status'], 'failed')
|
||
self.assertEqual(self.state('2026-11-06')['attempts'], 1)
|
||
self.assertEqual(self.state()['attempts'], 3)
|
||
|
||
def test_corrupt_attempt_state_does_not_reset_budget(self):
|
||
path = self.root / 'recovery' / '2026-11-05.json'
|
||
pv.atomic(path, {'attempts': 'unknown'})
|
||
before = path.read_bytes()
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'blocked_state')
|
||
self.assertEqual(path.read_bytes(), before)
|
||
self.build.assert_not_called()
|
||
|
||
def test_configuration_cannot_expand_safety_limits(self):
|
||
self.build.side_effect = RuntimeError('构建失败')
|
||
with patch.object(config, 'PLAN_RECOVERY_MAX_ATTEMPTS', 100), patch.object(config, 'PLAN_RECOVERY_COOLDOWN_SECONDS', 1):
|
||
for minutes in (0, 15, 30):
|
||
self.assertEqual(recovery.ensure_today(now=NOW + dt.timedelta(minutes=minutes))['status'], 'failed')
|
||
self.assertEqual(recovery.ensure_today(now=NOW + dt.timedelta(minutes=45))['status'], 'attempts_exhausted')
|
||
self.assertEqual(self.build.call_count, 3)
|
||
|
||
def test_disabled_recovery_does_not_construct(self):
|
||
with patch.object(config, 'PLAN_RECOVERY_MAX_ATTEMPTS', 0):
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'attempts_exhausted')
|
||
self.build.assert_not_called()
|
||
|
||
def test_source_changes_after_build_do_not_publish(self):
|
||
self.ready.side_effect = [DAY, DAY, RuntimeError('来源日期变化')]
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'failed')
|
||
self.build.assert_called_once()
|
||
self.generate.assert_not_called()
|
||
self.assertFalse((self.root / ('plan_' + DAY + '.json')).exists())
|
||
|
||
def test_build_finishes_after_window_does_not_publish(self):
|
||
before = dt.datetime.fromisoformat('2026-11-05T15:09:00+08:00')
|
||
after = dt.datetime.fromisoformat('2026-11-05T15:11:00+08:00')
|
||
with patch.object(recovery, '_now', side_effect=[before, before, after, after]):
|
||
self.assertEqual(recovery.ensure_today()['status'], 'failed')
|
||
self.generate.assert_not_called()
|
||
|
||
def test_plan_failure_preserves_previous_snapshot(self):
|
||
self.publish('2026-11-03', cutoff=NOW - dt.timedelta(days=1))
|
||
path = self.root / 'plan_2026-11-03.json'
|
||
before = path.read_bytes()
|
||
self.generate.side_effect = SystemExit('采集失败')
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'failed')
|
||
self.assertEqual(path.read_bytes(), before)
|
||
self.assertEqual(self.state()['attempts'], 1)
|
||
|
||
def test_published_plan_with_later_failure_is_not_rebuilt(self):
|
||
def partly_published(*args, **kwargs):
|
||
self.publish()
|
||
raise OSError('Markdown 写入失败')
|
||
self.generate.side_effect = partly_published
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'failed')
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'already_ready')
|
||
self.build.assert_called_once()
|
||
|
||
def test_collection_finishes_after_window_does_not_publish(self):
|
||
before = dt.datetime.fromisoformat('2026-11-05T15:09:00+08:00')
|
||
after = dt.datetime.fromisoformat('2026-11-05T15:11:00+08:00')
|
||
with patch.object(recovery, '_now', side_effect=[before, before, before, after, after]):
|
||
self.assertEqual(recovery.ensure_today()['status'], 'failed')
|
||
self.assertFalse((self.root / ('plan_' + DAY + '.json')).exists())
|
||
self.assertEqual(self.state()['attempts'], 1)
|
||
|
||
def test_readiness_changes_during_collection_does_not_publish(self):
|
||
self.ready.side_effect = [DAY, DAY, DAY, RuntimeError('发布前热度日期失效')]
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'failed')
|
||
self.assertFalse((self.root / ('plan_' + DAY + '.json')).exists())
|
||
|
||
def test_real_generate_checks_recovery_boundary_before_any_publication(self):
|
||
import user_candidates
|
||
guard = Mock(side_effect=RuntimeError('已过恢复时窗'))
|
||
data = {'date': DAY, 'main': [], 'observe': [], 'plan_version': 'isolated'}
|
||
with patch.object(user_candidates, 'active', return_value=[]), \
|
||
patch.object(plan, 'collect', return_value=data), \
|
||
patch.object(plan.sources, 'market_context', return_value={}), \
|
||
patch.object(plan, 'write_published_md') as markdown, pv.writer(self.root):
|
||
with self.assertRaisesRegex(RuntimeError, '已过恢复时窗'):
|
||
GENERATE(DAY, 20, 10, 5, before_publish=guard)
|
||
guard.assert_called_once_with(DAY)
|
||
markdown.assert_not_called()
|
||
self.assertFalse((self.root / ('plan_' + DAY + '.json')).exists())
|
||
self.assertFalse((self.root / 'versions' / DAY).exists())
|
||
|
||
def test_recovery_keeps_real_cross_process_lock_through_build_and_publish(self):
|
||
def assert_locked():
|
||
with open(self.root / '.publish.lock', 'a') as other:
|
||
with self.assertRaises(BlockingIOError):
|
||
fcntl.flock(other, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
def build(*args):
|
||
assert_locked()
|
||
return DAY
|
||
def publish(*args, **kwargs):
|
||
assert_locked()
|
||
return self.publish(*args, **kwargs)
|
||
self.build.side_effect = build
|
||
self.generate.side_effect = publish
|
||
self.assertEqual(recovery.ensure_today(now=NOW)['status'], 'recovered')
|
||
|
||
def test_cli_syncs_existing_workflow_once_then_refreshes_recovered_plan(self):
|
||
refresh = types.SimpleNamespace(refresh=Mock(side_effect=[{'status': 'no_morning_today'}, {'status': 'published'}]))
|
||
research = types.SimpleNamespace(sync=Mock())
|
||
with patch.dict(sys.modules, {'plan_refresh': refresh, 'user_research': research}), \
|
||
patch.object(recovery, 'ensure_today', return_value={'status': 'recovered'}) as recover, \
|
||
patch.object(sys, 'argv', ['run.py', 'plan-refresh']):
|
||
run.main()
|
||
research.sync.assert_called_once_with()
|
||
recover.assert_called_once_with(None)
|
||
self.assertEqual(refresh.refresh.call_count, 2)
|
||
self.pool.assert_not_called()
|
||
|
||
def test_cli_does_not_treat_waiting_cooldown_or_failure_as_success(self):
|
||
for status in ('waiting_data', 'cooldown', 'attempts_exhausted', 'failed', 'blocked_state'):
|
||
refresh = types.SimpleNamespace(refresh=Mock(return_value={'status': 'no_morning_today'}))
|
||
research = types.SimpleNamespace(sync=Mock())
|
||
with self.subTest(status=status), patch.dict(sys.modules, {'plan_refresh': refresh, 'user_research': research}), \
|
||
patch.object(recovery, 'ensure_today', return_value={'status': status}), \
|
||
patch.object(sys, 'argv', ['run.py', 'plan-refresh']):
|
||
with self.assertRaisesRegex(SystemExit, status):
|
||
run.main()
|
||
research.sync.assert_called_once()
|
||
refresh.refresh.assert_called_once()
|
||
|
||
def test_cli_existing_plan_only_runs_normal_refresh(self):
|
||
refresh = types.SimpleNamespace(refresh=Mock(return_value={'status': 'unchanged'}))
|
||
research = types.SimpleNamespace(sync=Mock())
|
||
with patch.dict(sys.modules, {'plan_refresh': refresh, 'user_research': research}), \
|
||
patch.object(recovery, 'ensure_today') as recover, \
|
||
patch.object(sys, 'argv', ['run.py', 'plan-refresh']):
|
||
run.main()
|
||
recover.assert_not_called()
|
||
research.sync.assert_called_once()
|
||
|
||
def test_daily_build_and_recovery_share_publish_lock(self):
|
||
entered = []
|
||
@contextlib.contextmanager
|
||
def guard(root):
|
||
entered.append('lock')
|
||
yield
|
||
entered.append('unlock')
|
||
def build(*args):
|
||
self.assertEqual(entered, ['lock'])
|
||
return DAY
|
||
self.build.side_effect = build
|
||
with patch.object(pv, 'writer', side_effect=guard):
|
||
self.assertEqual(run.cmd_build('all', 'daily', None, None, None), DAY)
|
||
self.assertEqual(entered, ['lock', 'unlock'])
|
||
|
||
def test_regime_cli_propagates_unpublished_status(self):
|
||
for status in ('no_morning', 'no_morning_today', 'outside_window', 'published', 'unchanged'):
|
||
regime = types.SimpleNamespace(append_to_snapshot=Mock(return_value={'status': 'OK', 'publication_status': status}))
|
||
with self.subTest(status=status), patch.dict(sys.modules, {'regime': regime}), \
|
||
patch.object(sys, 'argv', ['run.py', 'regime-append', '--date', DAY]):
|
||
if status in ('published', 'unchanged'):
|
||
run.main()
|
||
else:
|
||
with self.assertRaisesRegex(SystemExit, status):
|
||
run.main()
|
||
|
||
|
||
if __name__ == '__main__':
|
||
unittest.main()
|