From ef873bbead17f0de21da9ceba53149b49145cc23 Mon Sep 17 00:00:00 2001 From: zlt Date: Mon, 19 Oct 2026 09:48:18 +0800 Subject: [PATCH] =?UTF-8?q?=E6=AF=8F=E6=97=A5=E8=AE=A1=E5=88=92=EF=BC=9A?= =?UTF-8?q?=E6=8B=A6=E6=88=AA=E4=B8=8A=E6=B8=B8=E6=97=A5=E6=9C=9F=E5=81=9C?= =?UTF-8?q?=E6=9B=B4=EF=BC=8C=E6=9E=84=E5=BB=BA=E5=A4=B1=E8=B4=A5=E9=98=BB?= =?UTF-8?q?=E6=AD=A2=E5=90=8E=E7=BB=AD=E5=8F=91=E5=B8=83=E4=B8=8E=E5=85=A5?= =?UTF-8?q?=E6=B1=A0=EF=BC=88=E5=8F=B0=E8=B4=A6=20074=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一、上游停更必须在写入前失败 10 月 19 日实测传导和热度仍是 10 月 15 日,收盘行情已有 10 月 16 日。 自动日更按北京时间上一交易日核对三路日期,读取失败不再猜今天。 二、调度成功必须代表所需步骤完成 部分因子或冻结失败返回失败,缺今日完整晨版的修订也返回失败。 本地 12 项专项及版本契约回归通过,完整依赖测试另在禁网候选镜像验证。 三、保留交易与历史边界 不修改排序、仓位、档位、交易时段及 PMS 过期保护。 显式历史维护仍可指定日期,不覆盖既有版本。 外部资金流和热度未恢复,本次不宣称已经生成新计划。 --- build_readiness.py | 48 +++++++++++++++++++++ docs/复盘决定台账.md | 9 ++++ run.py | 29 ++++++------- test_build_readiness.py | 96 +++++++++++++++++++++++++++++++++++++++++ test_xxl_trigger.py | 14 ++++++ 5 files changed, 181 insertions(+), 15 deletions(-) create mode 100644 build_readiness.py create mode 100644 test_build_readiness.py diff --git a/build_readiness.py b/build_readiness.py new file mode 100644 index 0000000..610c774 --- /dev/null +++ b/build_readiness.py @@ -0,0 +1,48 @@ +"""自动日更的日期核验。只读三路来源,不重建计划、不调用模型。""" +import datetime as dt +from zoneinfo import ZoneInfo + +import pandas as pd +import db + + +def expected_data_day(now=None): + from chinese_calendar import is_holiday + local = (now or dt.datetime.now(dt.timezone.utc)).astimezone(ZoneInfo('Asia/Shanghai')) + day = local.date() - dt.timedelta(days=1) + while day.weekday() >= 5 or is_holiday(day): + day -= dt.timedelta(days=1) + return day + + +def check(now=None): + """数据未齐则在任何写入前失败;不能用缺失数据推断今天休市。""" + expected = expected_data_day(now) + dates = {} + failures = [] + for name, source, sql in ( + ('传导', 'pg', 'SELECT MAX(scan_date) d FROM v_factor_transmission'), + ('收盘行情', 'price', 'SELECT MAX(`timestamp`) d FROM gp_day_data'), + ('资金热度', 'heat', 'SELECT MAX(trade_date) d FROM stock_fund_heat_scores'), + ): + try: + frame = db.read_pg(sql) if source == 'pg' else db.read_mysql(source, sql) + value = None if frame.empty else frame.iloc[0, 0] + day = None if value is None or pd.isna(value) else pd.Timestamp(value).date() + except Exception as exc: + failures.append(f'{name}读取失败({type(exc).__name__})') + continue + dates[name] = day + if day != expected: + failures.append(f'{name}日期 {day or "缺失"}') + if failures: + raise RuntimeError(f'日更输入未就绪:应为 {expected};' + ';'.join(failures) + + '。保留上次完整计划,停止构建及入池。') + return expected.isoformat() + + +if __name__ == '__main__': + try: + print('日更输入日期已对齐:' + check()) + except RuntimeError as exc: + raise SystemExit(str(exc)) diff --git a/docs/复盘决定台账.md b/docs/复盘决定台账.md index b38e6e0..b32485d 100644 --- a/docs/复盘决定台账.md +++ b/docs/复盘决定台账.md @@ -750,3 +750,12 @@ - **复核日期。** 本日完成30专项、44批离线回归后在155候选镜像验证并部署。旧已保存拒绝由正常扫描收尾,写前备份、写后读回;不制造交易或调用模型验收。 - **073部署补记。** PMS60de23e于14:39:51在155生效。30专项、44批本地及镜像禁网回归通过。14:40:01正常扫描撤下瑞芯微旧提议,唯一拒绝6858、零指令;14:45重复读回无重复收尾。浏览器待确认0条、错误为空。原参数和修复账务保持。仅既有WAIT时刻推进,未制造交易。拒绝时缺失的历史重问基线仍为空,不补造。 + + +## 074 · 2026-10-19 · 上游停更不能作为日更成功 + +- **日期。** 2026-10-19。 +- **改动。** 自动日更写入前,核对传导、收盘行情和资金热度是否为北京时间上一交易日。来源不可达或缺日期时失败,不猜今天。因子及冻结失败阻断后续计划和入池。盘中缺当日完整晨版时,修订命令返回失败。显式历史维护入口保留。 +- **依据。** 09:40 实测资金流、热度和传导停在 10-15,收盘行情已有 10-16。周六同步沿用旧资金流,周一 07:10 又构建旧日因子,平台仍报成功。PMS 正确拦截两交易日前计划。 +- **预期。** 数据停更在构建入口暴露,旧榜不再随成功链推入池。不改排序、档位、仓位、交易时段或 PMS 一交易日时效上限。不伪造当日晨版。 +- **复核日期。** 本轮合成回归及部署读回。外部 10-16 资金流和热度尚缺,恢复后另行核验完整链;不得把错误明确显示当作数据已恢复。 diff --git a/run.py b/run.py index 6e4334b..6b335d1 100644 --- a/run.py +++ b/run.py @@ -116,18 +116,9 @@ def cmd_register(): def _latest_data_day() -> str: - """daily 不传 --date 时的默认日:基座传导台账最新 scan_date(=数据日)。 - - 盘前链(07-30 D 案)在次日早晨补全上一交易日,"今天"多半还没有数据; - 以基座刚完成的扫描日为准,两边永远对齐。查不到再退回今天。""" - try: - df = db.read_pg("SELECT MAX(scan_date) d FROM v_factor_transmission") - v = None if df.empty else df.iloc[0, 0] - if v is not None and not pd.isna(v): - return pd.Timestamp(v).date().isoformat() - except Exception as e: # noqa: BLE001 —— 视图不可达时退回今天,不阻塞构建 - print(f" (取基座最新扫描日失败,默认改用今天: {e!r})") - return dt.date.today().isoformat() + """自动日更必须取得上一交易日的完整日期锚;失败不猜日期。""" + from build_readiness import check + return check() def cmd_build(which, mode, start, end, date, do_freeze=True): @@ -138,6 +129,7 @@ def cmd_build(which, mode, start, end, date, do_freeze=True): raise SystemExit("history 模式需要 --start 与 --end") codes = list(factors.FACTORS) if which == "all" else [which] frames = {} + failures = [] for code in codes: if code not in factors.BUILDERS: raise SystemExit(f"未知因子: {code}(可选: {list(factors.FACTORS)} 或 all)") @@ -148,14 +140,18 @@ def cmd_build(which, mode, start, end, date, do_freeze=True): common.write_factor(factors.FACTORS[code], df, mode) except Exception as e: # noqa: BLE001 —— 一路失败不拖累其余(批量容错) print(f" ❌ {code} 失败: {e!r}") + failures.append(code) + # 各因子仍独立尝试,但部分成功不能让调度继续计划与入池。 + if failures: + raise RuntimeError('因子构建失败:' + '、'.join(failures)) # 日更顺手冻结:输入与输出落在同一目录,任何一行因子值都能被逐步复算。 # history 模式默认不冻结(逐日冻结应单独跑,避免一次回填写出几百个目录)。 if do_freeze and mode == "daily": try: import freeze freeze.snapshot(start, extra_frames=frames) - except Exception as e: # noqa: BLE001 —— 冻结失败不该让因子构建算失败 - print(f" ❌ 冻结失败(因子已落库): {e!r}") + except Exception as e: # noqa: BLE001 —— 因子已写,但完整输入缺失不得继续发布 + raise RuntimeError("输入冻结失败,停止计划及入池(因子已落库)") from e def main(): @@ -237,7 +233,10 @@ def main(): regime.append_to_snapshot(day) elif a.cmd == 'plan-refresh': import plan_refresh - print(plan_refresh.refresh(a.date)) + result = plan_refresh.refresh(a.date) + print(result) + if result['status'] in ('no_morning', 'no_morning_today'): + raise SystemExit('今日完整晨版缺失,保留上次版本,修订未执行。') elif a.cmd == "judgement-snapshot": import judgement judgement.snapshot(a.date) diff --git a/test_build_readiness.py b/test_build_readiness.py new file mode 100644 index 0000000..d18be77 --- /dev/null +++ b/test_build_readiness.py @@ -0,0 +1,96 @@ +"""本地离线:python -m unittest test_build_readiness;来源与写入全部替身。""" +import datetime as dt +import socket +import sys +import types +import unittest +from unittest.mock import patch +import pandas as pd +for name in ('psycopg', 'pymysql'): + try: + __import__(name) + except ImportError: + sys.modules[name] = types.ModuleType(name) +import build_readiness as readiness +import run + +NOW = dt.datetime.fromisoformat('2026-10-19T01:40:00+00:00') + +class ReadinessTest(unittest.TestCase): + def setUp(self): + self.net = patch.object(socket.socket, 'connect', side_effect=AssertionError('禁止联网')) + self.net.start() + self.addCleanup(self.net.stop) + + def test_monday_uses_friday(self): + self.assertEqual(readiness.expected_data_day(NOW), dt.date(2026, 10, 16)) + + def test_beijing_midnight_not_utc_date(self): + now = dt.datetime.fromisoformat('2026-10-18T17:50:00+00:00') + self.assertEqual(readiness.expected_data_day(now), dt.date(2026, 10, 16)) + + def test_holiday_and_weekend(self): + with patch('chinese_calendar.is_holiday', side_effect=lambda d: dt.date(2026, 10, 1) <= d <= dt.date(2026, 10, 7)): + now = dt.datetime.fromisoformat('2026-10-08T07:10:00+08:00') + self.assertEqual(readiness.expected_data_day(now), dt.date(2026, 9, 30)) + + def test_aligned_sources_pass(self): + frame = pd.DataFrame({'d': [dt.date(2026, 10, 16)]}) + with patch.object(readiness.db, 'read_pg', return_value=frame), patch.object(readiness.db, 'read_mysql', return_value=frame): + self.assertEqual(readiness.check(NOW), '2026-10-16') + + def test_stale_and_empty_sources_fail(self): + for value in ['2026-10-15', None, pd.NaT, '2026-10-19']: + with self.subTest(value=value), patch.object(readiness.db, 'read_pg', return_value=pd.DataFrame({'d': [value]})), patch.object(readiness.db, 'read_mysql', return_value=pd.DataFrame({'d': ['2026-10-16']})): + with self.assertRaisesRegex(RuntimeError, '应为 2026-10-16.*传导日期'): + readiness.check(NOW) + + def test_missing_heat_fails_even_if_scan_current(self): + with patch.object(readiness.db, 'read_pg', return_value=pd.DataFrame({'d': ['2026-10-16']})), patch.object(readiness.db, 'read_mysql', side_effect=lambda source, sql: pd.DataFrame({'d': ['2026-10-15' if source == 'heat' else '2026-10-16']})): + with self.assertRaisesRegex(RuntimeError, '资金热度日期 2026-10-15'): + readiness.check(NOW) + + def test_unavailable_source_does_not_guess_today(self): + with patch.object(readiness.db, 'read_pg', side_effect=OSError('secret-connection')), patch.object(readiness.db, 'read_mysql', return_value=pd.DataFrame({'d': ['2026-10-16']})): + with self.assertRaisesRegex(RuntimeError, '传导读取失败') as result: + readiness.check(NOW) + self.assertNotIn('secret-connection', str(result.exception)) + + def test_stale_default_stops_before_any_writes(self): + with patch.object(readiness, 'check', side_effect=RuntimeError('上游停更')), patch.object(run.common, 'write_factor') as write: + with self.assertRaisesRegex(RuntimeError, '上游停更'): + run.cmd_build('all', 'daily', None, None, None) + write.assert_not_called() + + def test_explicit_history_keeps_existing_entry(self): + with patch.object(readiness, 'check', side_effect=AssertionError('不限制显式维护日期')), patch.object(run.factors, 'BUILDERS', {'test': lambda a,b: pd.DataFrame()}), patch.object(run.factors, 'FACTORS', {'test': 'test_table'}), patch.object(run.common, 'write_factor') as write: + run.cmd_build('test', 'daily', None, None, '2026-10-15', do_freeze=False) + write.assert_called_once() + + def test_partial_build_failure_is_not_success(self): + def fail(a,b): raise ValueError('failed') + frozen=types.ModuleType('freeze'); frozen.snapshot=lambda *a,**kw: self.fail('不可冻结部分结果') + with patch.dict(sys.modules, {'freeze': frozen}), patch.object(run.factors, 'BUILDERS', {'good': lambda a,b: pd.DataFrame(), 'bad': fail}), patch.object(run.factors, 'FACTORS', {'good': 'g', 'bad': 'b'}), patch.object(run.common, 'write_factor') as write: + with self.assertRaisesRegex(RuntimeError, '因子构建失败:bad'): + run.cmd_build('all', 'daily', None, None, '2026-10-16') + write.assert_called_once() + + def test_freeze_failure_is_not_success(self): + frozen=types.ModuleType('freeze') + def fail(*a, **kw): raise OSError('disk full') + frozen.snapshot=fail + with patch.dict(sys.modules, {'freeze': frozen}), patch.object(run.factors, 'BUILDERS', {'test': lambda a,b: pd.DataFrame()}), patch.object(run.factors, 'FACTORS', {'test': 't'}), patch.object(run.common, 'write_factor'): + with self.assertRaisesRegex(RuntimeError, '输入冻结失败'): + run.cmd_build('test', 'daily', None, None, '2026-10-16') + + def test_refresh_missing_morning_fails_cli(self): + mod=types.ModuleType('plan_refresh') + for status in ('no_morning', 'no_morning_today', 'outside_window', 'unchanged'): + mod.refresh=lambda *a, **kw: {'status': status} + with self.subTest(status=status), patch.dict(sys.modules, {'plan_refresh': mod}), patch.object(sys, 'argv', ['run.py','plan-refresh']): + if status.startswith('no_morning'): + with self.assertRaisesRegex(SystemExit, '今日完整晨版缺失'): run.main() + else: run.main() + +if __name__ == '__main__': + unittest.main() diff --git a/test_xxl_trigger.py b/test_xxl_trigger.py index c299704..80a90e4 100644 --- a/test_xxl_trigger.py +++ b/test_xxl_trigger.py @@ -100,6 +100,20 @@ def _(): assert "handleCode=500" in body and "plan" in body, body +@case("上游日期停更使 build 失败:不出计划、不入池,回调失败") +def _(): + ran, sent = _reset() + def stale(cmd): + ran.append(list(cmd)) + return 1, "日更输入未就绪:应为 2026-10-16;传导日期 2026-10-15" + xxl._run_step = stale + result = _trigger(callbackUrl="http://xxl/stale") + job = _wait_done(result["task_id"]) + assert job["state"] == "failed", job + assert [c[1] for c in ran] == ["build"], ran + assert len(sent) == 1 and b"handleCode=500" in sent[0][1], sent + + @case("同一时刻只允许一条链: 在跑时再触发 → 409") def _(): _reset()