From 0b3c50ef3478ccc415d5f81803fa5d8f7e93261d Mon Sep 17 00:00:00 2001 From: baobao Date: Tue, 3 Nov 2026 16:24:44 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A1=A5=E9=BD=90=E5=BD=93=E6=97=A5=E9=99=90?= =?UTF-8?q?=E5=88=B6=E4=BB=B7=E5=B9=B6=E9=9A=94=E7=A6=BB=E9=87=87=E9=9B=86?= =?UTF-8?q?=E4=B8=8E=E4=BA=A4=E6=98=93=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 3 + .env.example | 6 +- .gitignore | 3 + DEVLOG.md | 8 + app/core/execution_policy.py | 6 +- app/core/price_limits.py | 153 +++++++++++++ app/market_data_worker.py | 48 +++++ app/services/market.py | 18 +- app/services/param_store.py | 34 ++- app/services/price_limit_collector.py | 250 ++++++++++++++++++++++ app/services/price_limit_provider.py | 109 ++++++++++ app/services/price_limit_targets.py | 132 ++++++++++++ app/services/price_limits.py | 62 ++++++ app/services/rotation_service.py | 6 +- config/settings.py | 16 ++ docker-compose.yml | 24 ++- docs/开发入口.md | 2 +- docs/当日限制价采集边界_2026-11-03.md | 29 +++ scripts/run_tests.py | 8 +- scripts/test_market_data_worker.py | 99 +++++++++ scripts/test_price_limit_cache.py | 237 +++++++++++++++++++++ scripts/test_price_limit_collector.py | 295 ++++++++++++++++++++++++++ scripts/test_price_limit_provider.py | 234 ++++++++++++++++++++ scripts/test_price_limit_targets.py | 230 ++++++++++++++++++++ scripts/test_price_limits_core.py | 210 ++++++++++++++++++ 25 files changed, 2210 insertions(+), 12 deletions(-) create mode 100644 app/core/price_limits.py create mode 100644 app/market_data_worker.py create mode 100644 app/services/price_limit_collector.py create mode 100644 app/services/price_limit_provider.py create mode 100644 app/services/price_limit_targets.py create mode 100644 app/services/price_limits.py create mode 100644 docs/当日限制价采集边界_2026-11-03.md create mode 100644 scripts/test_market_data_worker.py create mode 100644 scripts/test_price_limit_cache.py create mode 100644 scripts/test_price_limit_collector.py create mode 100644 scripts/test_price_limit_provider.py create mode 100644 scripts/test_price_limit_targets.py create mode 100644 scripts/test_price_limits_core.py diff --git a/.dockerignore b/.dockerignore index 32a67b7..eb96f0b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,9 @@ __pycache__ *.pyc .env +.env.* +!.env.example +backups/ logs .DS_Store # 开发代理的本地配置和嵌套工作树不进入运行镜像。 diff --git a/.env.example b/.env.example index 1aca85e..05b22d6 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,10 @@ # 业务参数 (PMS_TOTAL_SCALE 等) 不必写在这里 —— settings.py 有初值, # 上线后经管理页面调整并持久化到 pms_runtime_param 表。 +# 限制价采集密钥PMS_TUSHARE_TOKEN必须仅写入专用.env.market-data,不要放进主.env。 +# 该专用文件只由pms-market-data服务注入,不提交Git,也不放进镜像。 +# 缓存消费和采集开关默认关闭,业务运行值经ParamStore维护。 + # ---- 153 代理 (pms_* 全部表 + trading_* + strategy_daily_results, 严格单表访问) ---- PROXY_DB_URL=mysql+pymysql://用户:密码@192.168.16.153:3307/my_quant_db @@ -60,4 +64,4 @@ PMS_QMT_SIGN_SEED_HEX= PMS_QMT_PEER_PUBKEY_B64= -PMS_QMT_WS_URL=ws://192.168.18.182:9443/pms \ No newline at end of file +PMS_QMT_WS_URL=ws://192.168.18.182:9443/pms diff --git a/.gitignore b/.gitignore index ec858a0..a87a76e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ # 机密: 数据库/Redis 连接串与密码, 绝不入库 (服务器上手工维护, 参照 .env.example) .env +.env.* +!.env.example +backups/ __pycache__/ *.pyc diff --git a/DEVLOG.md b/DEVLOG.md index e879785..fb41237 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2721,3 +2721,11 @@ PMS最终代码56db0fd已推送,155 web/WS/beat为execution-2b1c447,worker 15:38新v3快照50首选100候选、信号与名单就绪,数据11-02/执行11-03,frozen=false;11-04版待夜间数据,不回填早盘冻结。15:40确认1598996、最近160条连续、资金10.3秒、数量现金一致,无待入账和孤儿。已安排11-04 10:00一次性任务v3核验自然请求,未恢复长期观察。 现行交易所规则与源表核实后,日线计算评估为普通情形可行,但缺完整目标日公司行为与证券状态,不能作为唯一精确兜底。发现11-04已有明确除息个例,报价与名称资料陈旧,故本轮未将推算接入下单;保留正式当日限制价及逐票等待。该评估和真实数据日期、回退备份、当前参数及验证安排已写发布接续和决策台账。 + +## 2026-11-03 当日限制价独立采集代码完成 + +按用户批准方案实现155模拟PMS的独立采集、本地当日缓存和两个执行消费入口。实际Tushare单证券HTTPS可取得11月3日记录,逗号批量代码返回空,故采用逐票限速。目标覆盖原策略全部持仓、活动指令、在途、计划和建议,以及微盘完整100候选;入口排除北交所。查询共享gp_stk_limit的pre_close列存在,但11月2日实际样本为空,因此该票需要Tushare补全证据,不能冒充已完整校验。 + +新增缓存及采集开关默认关闭,密钥仅专用采集容器注入。价格严格校验、跨日失效、逐票隔离、重试退避、租约防并发、来源版本留痕均已实现;日线推算不进入下单。只读复核确认本次不需重建WS或beat,待发布目标为worker、web和新增采集进程。 + +开发机新增121项隔离检查通过,最终全量末行ALL SUITES PASS,日志/private/tmp/pms-limits-local-tests-release-20261103.log。审查补齐共享坏值重复退避、超时票饥饿、单票脏元数据及开关损坏四类边界。后续以已提交代码构建候选并完成155断网全量,再启动采集和核对实际覆盖;本条不代表已部署或自然交易判收。 diff --git a/app/core/execution_policy.py b/app/core/execution_policy.py index af2e03e..651443a 100644 --- a/app/core/execution_policy.py +++ b/app/core/execution_policy.py @@ -106,6 +106,10 @@ def price_order(*, side, quote, now=None, user_limit=None, anchor_price=None, pa value = float(price) if not low <= value <= high: return wait('用户边界或报价超出当日合法价格范围') - return {'action': 'FIRE', 'limit_price': value, 'reason': note, + result = {'action': 'FIRE', 'limit_price': value, 'reason': note, 'degraded': minute, 'quote_source': source, 'quote_as_of': at.isoformat(), 'quote_age_sec': age, 'anchor_price': anchor, 'user_limit': boundary or None} + if quote.get('limit_version'): + result.update(limit_source=quote.get('limit_source'), limit_version=quote['limit_version'], + limit_fetched_at=quote.get('limit_fetched_at')) + return result diff --git a/app/core/price_limits.py b/app/core/price_limits.py new file mode 100644 index 0000000..011c535 --- /dev/null +++ b/app/core/price_limits.py @@ -0,0 +1,153 @@ +"""限制价主源的纯校验。只接受目标日沪深A股事实,不计算或补造价格。""" +from collections.abc import Mapping +from datetime import date, datetime +from decimal import Decimal, InvalidOperation, localcontext +import hashlib +import json +import re + +MAX_ROWS = 6000 +REQUIRED_FIELDS = ('trade_date', 'ts_code', 'pre_close', 'up_limit', 'down_limit') +_CENT = Decimal('0.01') +_SUPPORTED = re.compile(r'(?:60[0-9]{4}\.SH|68[0-9]{4}\.SH|00[0-9]{4}\.SZ|30[0-9]{4}\.SZ)') +_IDENTIFIABLE = re.compile(r'[0-9]{6}\.(?:SH|SZ|BJ)') + + +def supported(code): + """仅规范化大小写与首尾空白,不猜测交易所,不包含北交所。""" + return isinstance(code, str) and _SUPPORTED.fullmatch(code.strip().upper()) is not None + + +def _result(error=None): + return {'records': {}, 'invalid': {}, 'errors': [error] if error else []} + + +def _day(value): + if isinstance(value, datetime): + return value.date().isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, bool) or not isinstance(value, (str, int)): + raise ValueError('INVALID_DATE') + value = str(value).strip() + if re.fullmatch(r'[0-9]{8}', value): + value = value[:4]+'-'+value[4:6]+'-'+value[6:] + elif not re.fullmatch(r'[0-9]{4}-[0-9]{2}-[0-9]{2}', value): + raise ValueError('INVALID_DATE') + try: + return date.fromisoformat(value).isoformat() + except ValueError: + raise ValueError('INVALID_DATE') from None + + +def _price(value): + if isinstance(value, bool) or not isinstance(value, (str, int, float, Decimal)): + raise ValueError('INVALID_PRICE') + text = str(value).strip() + if not text or len(text) > 128: + raise ValueError('INVALID_PRICE') + try: + number = Decimal(text) + if not number.is_finite() or number < 0: + raise ValueError('INVALID_PRICE') + with localcontext() as context: + context.prec = 28 + context.traps[InvalidOperation] = True + cents = number.quantize(_CENT) + if cents != number: + raise ValueError('INVALID_PRECISION') + return cents + except InvalidOperation: + raise ValueError('INVALID_PRICE') from None + + +def normalize_rows(rows, day, requested_codes=None): + """返回可JSON序列化记录;坏票隔离,结构错误拒绝整批。 + + 日期输出ISO格式,价格输出精确两位字符串。同票任何坏行都会隔离该票, + 不能用另一条正确日期或正确价格的行掩盖。错误仅返回固定原因码。 + """ + try: + target = _day(day) + except ValueError: + return _result('INVALID_TARGET_DATE') + if not isinstance(rows, (list, tuple)): + return _result('INVALID_ROWS') + if len(rows) > MAX_ROWS: + return _result('TOO_MANY_ROWS') + if any(not isinstance(row, Mapping) for row in rows): + return _result('INVALID_ROW_STRUCTURE') + allowed = None + if requested_codes is not None: + if not isinstance(requested_codes, (list, tuple, set, frozenset)): + return _result('INVALID_REQUESTED_CODES') + if any(not supported(code) for code in requested_codes): + return _result('INVALID_REQUESTED_CODES') + allowed = {code.strip().upper() for code in requested_codes} or None + result = _result() + records, invalid = result['records'], result['invalid'] + for index, row in enumerate(rows): + raw_code = row.get('ts_code') + code = raw_code.strip().upper() if isinstance(raw_code, str) else '' + # 不回显任意上游文本;无法识别为证券代码时只记录行号。 + key = code if _IDENTIFIABLE.fullmatch(code) else f'row:{index+1}' + reason = None + if not supported(code): + reason = 'OUT_OF_SCOPE' if _IDENTIFIABLE.fullmatch(code) else 'INVALID_CODE' + elif allowed is not None and code not in allowed: + reason = 'NOT_REQUESTED' + elif any(field not in row for field in REQUIRED_FIELDS): + reason = 'MISSING_FIELD' + else: + try: + if _day(row['trade_date']) != target: + raise ValueError('DATE_MISMATCH') + close, high, low = (_price(row[field]) for field in ('pre_close', 'up_limit', 'down_limit')) + if high == 0 or low == 0: + raise ValueError('UNSUPPORTED') + if close <= 0: + raise ValueError('INVALID_PRE_CLOSE') + if low >= high: + raise ValueError('INVALID_RANGE') + if not low <= close <= high: + raise ValueError('PRE_CLOSE_OUTSIDE_LIMITS') + record = {'trade_date': target, 'ts_code': code, 'pre_close': format(close, '.2f'), + 'up_limit': format(high, '.2f'), 'down_limit': format(low, '.2f')} + digest = hashlib.sha256(json.dumps(record, sort_keys=True, ensure_ascii=False, + separators=(',', ':')).encode('utf-8')).hexdigest() + record['version'] = digest + if code in records and records[code] != record: + raise ValueError('DUPLICATE_CONFLICT') + if code not in invalid: + records[code] = record + except ValueError as exc: + reason = str(exc) + if reason: + records.pop(code, None) + invalid.setdefault(key, reason) + return result + + +def provider_decode(body, day, requested_codes=None): + """校验Tushare JSON封套。HTTP状态由调用层检查,不输出msg或响应正文。""" + if not isinstance(body, Mapping) or type(body.get('code')) is not int: + return _result('INVALID_PROVIDER_RESPONSE') + if body['code'] != 0: + return _result('PROVIDER_ERROR') + data = body.get('data') + if not isinstance(data, Mapping): + return _result('INVALID_PROVIDER_DATA') + fields, items = data.get('fields'), data.get('items') + if (not isinstance(fields, list) or not fields + or any(not isinstance(field, str) or not field for field in fields) + or len(set(fields)) != len(fields)): + return _result('INVALID_PROVIDER_FIELDS') + if any(field not in fields for field in REQUIRED_FIELDS): + return _result('MISSING_PROVIDER_FIELDS') + if not isinstance(items, list): + return _result('INVALID_PROVIDER_ITEMS') + if len(items) > MAX_ROWS: + return _result('TOO_MANY_ROWS') + if any(not isinstance(item, (list, tuple)) or len(item) != len(fields) for item in items): + return _result('INVALID_PROVIDER_ITEM_STRUCTURE') + return normalize_rows([dict(zip(fields, item)) for item in items], day, requested_codes) diff --git a/app/market_data_worker.py b/app/market_data_worker.py new file mode 100644 index 0000000..15d025d --- /dev/null +++ b/app/market_data_worker.py @@ -0,0 +1,48 @@ +"""当日限制价的独立采集入口,不启动Celery或QMT连接。""" +import logging +import re +import signal +from threading import Event + + +logger = logging.getLogger('pms.market_data_worker') + + +def _summary(result): + """只保留固定摘要字段,源响应及异常正文不得进入进程日志。""" + if not isinstance(result, dict): + raise ValueError('采集摘要格式无效') + status = result.get('status') + status = status if isinstance(status, str) and re.fullmatch(r'[A-Za-z0-9_-]{1,48}', status) else 'unknown' + day = result.get('day') + day = day if isinstance(day, str) and re.fullmatch(r'\d{4}-\d{2}-\d{2}', day) else '-' + counts = [] + for key in ('target_count', 'valid_count', 'missing_count', 'error_count'): + value = result.get(key) + counts.append(value if type(value) is int and value >= 0 else 0) + return status, day, *counts + + +def main(): + from app.services import price_limit_collector + + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s %(message)s') + stopping = Event() + + def stop(_signum, _frame): + stopping.set() + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + while not stopping.is_set(): + try: + result = price_limit_collector.run_once() + logger.info('限制价采集 status=%s day=%s target_count=%s valid_count=%s missing_count=%s error_count=%s', + *_summary(result)) + except Exception as exc: + logger.warning('限制价采集失败:%s', type(exc).__name__) + stopping.wait(10) + + +if __name__ == '__main__': + main() diff --git a/app/services/market.py b/app/services/market.py index cae3d69..46a9f79 100644 --- a/app/services/market.py +++ b/app/services/market.py @@ -136,14 +136,22 @@ def execution_quote(code, now=None, limits=None): except (ValueError, TypeError): return {} if limits is None: - rows = fetch_all('SELECT up_limit,down_limit FROM gp_stk_limit ' - 'WHERE trade_date=:day AND ts_code=:code', - {'day': current.date().isoformat(), 'code': code}) - limits = rows[0] if rows else {} + if param_store.get_bool('PMS_LIMIT_CACHE_ENABLED', False): + from app.services import price_limits + limits = price_limits.get_limits(current.date(), [code]).get(code, {}) + else: + rows = fetch_all('SELECT up_limit,down_limit FROM gp_stk_limit ' + 'WHERE trade_date=:day AND ts_code=:code', + {'day': current.date().isoformat(), 'code': code}) + limits = rows[0] if rows else {} return {'kind': 'minute', 'source': 'tushare:rt_min:1MIN', 'as_of': at.isoformat(), 'last': price, 'bid': None, 'ask': None, 'limit_up': (limits or {}).get('up_limit'), - 'limit_down': (limits or {}).get('down_limit'), 'halted': None} + 'limit_down': (limits or {}).get('down_limit'), 'halted': None, + 'limit_source': (limits or {}).get('source') or ('gp_stk_limit' if + (limits or {}).get('up_limit') and (limits or {}).get('down_limit') else None), + 'limit_version': (limits or {}).get('version'), + 'limit_fetched_at': (limits or {}).get('fetched_at')} # ================================================================ 参考位 def _shard_tables(months: int = 4) -> list: diff --git a/app/services/param_store.py b/app/services/param_store.py index e8c8dff..0b21e01 100644 --- a/app/services/param_store.py +++ b/app/services/param_store.py @@ -36,10 +36,28 @@ INFRA_PREFIX = ("PROXY_DB", "SOURCE_DB", "DB_MYSQL", "SIGNAL_REDIS", "PMS_REDIS" # 2026-09-03 补进会话票签名密钥 PMS_SESSION_SECRET: 它 2026-08 随登录一起加进 settings, # 当时漏了这份名单, 于是参数设置页把它当普通业务参数原文显示出来 (登录中间件读的是 # settings, 功能没受影响, 但密钥不该出现在页面上)。 -SECRET_KEYS = ("PMS_QMT_SIGN_SEED_HEX", "PMS_QMT_PEER_PUBKEY_B64", "PMS_SESSION_SECRET", "PMS_PLAN_WRITE_KEY") +SECRET_KEYS = ("PMS_QMT_SIGN_SEED_HEX", "PMS_QMT_PEER_PUBKEY_B64", "PMS_SESSION_SECRET", "PMS_PLAN_WRITE_KEY", "PMS_TUSHARE_TOKEN") +LIMIT_FLAGS = ('PMS_LIMIT_CACHE_ENABLED', 'PMS_LIMIT_COLLECT_ENABLED') + + +def _valid_limit_flag(value): + return isinstance(value, bool) or (type(value) in (str, int) and + str(value).strip().lower() in ('1', 'true', 'yes', 'on', 'y', '0', 'false', 'no', 'off', 'n')) # 运行态开关: key -> (默认值, 类型, 说明) RUNTIME_EXTRA = { + 'PMS_LIMIT_CACHE_ENABLED': (False, bool, '执行报价读取已核实的当日限制价缓存;缺失时逐票等待'), + 'PMS_LIMIT_COLLECT_ENABLED': (False, bool, '独立采集进程补齐目标证券的当日限制价,不生成交易指令'), + 'PMS_LIMIT_FETCH_TIMEOUT_SEC': (5, int, '限制价来源单次请求超时,秒'), + 'PMS_LIMIT_ROUND_BUDGET_SEC': (30, int, '每轮限制价HTTP采集阶段预算,秒'), + 'PMS_LIMIT_ROUND_MAX_CODES': (25, int, '每轮最多请求的目标证券数'), + 'PMS_LIMIT_MIN_REQUEST_INTERVAL_SEC': (1.2, float, '独立采集进程全部请求之间的最小间隔,秒'), + 'PMS_LIMIT_RETRY_BASE_SEC': (60, int, '限制价采集失败后的首次退避秒数'), + 'PMS_LIMIT_RETRY_MAX_SEC': (900, int, '限制价连续失败的最大退避秒数'), + 'PMS_LIMIT_MAX_FAILURES_PER_DAY': (32, int, '每只证券每日最多连续失败次数,成功清零;达到后停止当天重试'), + 'PMS_LIMIT_REFRESH_SEC': (900, int, '已有效限制价的再次核验间隔,秒'), + 'PMS_LIMIT_COLLECT_START': ('08:55', str, '限制价采集开始时间,北京时间HH:MM;无效时不采集'), + 'PMS_LIMIT_COLLECT_END': ('18:00', str, '限制价采集结束时间,北京时间HH:MM;仅交易日且日历可用时采集'), 'PMS_EXEC_REJECT_COOLDOWN_SEC': (60, int, '拒单后的首次退避秒数,不限制正常改价'), 'PMS_EXEC_REJECT_MAX_COOLDOWN_SEC': (900, int, '重复拒单退避最大秒数'), 'PMS_EXEC_REPRICE_ENABLED': (False, bool, '已授权委托按新鲜行情撤单确认后重挂;先发布上游执行依据接口'), @@ -186,6 +204,8 @@ RUNTIME_EXTRA = { # 冷启、连库超时), 全局暂停买入、休假模式、组合刹车就一起静默解除, 且没有任何调用方 # 看得出这是降级。安全开关的默认方向必须是"拦", 不是"放"。 FAIL_CLOSED = { + "PMS_LIMIT_CACHE_ENABLED": True, # 无法确认开关时仍走严格本地缓存,不退到旧读取。 + "PMS_LIMIT_COLLECT_ENABLED": False, "PMS_GLOBAL_BUY_HALT": True, # 读不到 → 当作已暂停买入 "PMS_GLOBAL_EXEC_HALT": True, # 读不到 → 当作已暂停执行 # 自主新建仓的默认值是 full (无人值守地开新仓), 所以它读不到时的安全方向格外要紧: @@ -462,6 +482,8 @@ def get(key: str, default=None): "宁可多拦一轮也不放行", key, _cache["error"], FAIL_CLOSED[key]) return FAIL_CLOSED[key] if key in rows: + if key in LIMIT_FLAGS and not _valid_limit_flag(rows[key].get('param_value')): + return FAIL_CLOSED[key] try: return _coerce(rows[key]["param_value"], t) except (TypeError, ValueError): @@ -510,6 +532,8 @@ def get_tuple_floats(key, default=(0.5, 0.25, 0.25)): def set_param(key: str, value, updated_by: str = "user") -> dict: """页面改参入口。校验键名与类型后落表, 并立即失效缓存。""" editable = _editable_keys() + if key in LIMIT_FLAGS and not _valid_limit_flag(value): + return {'ok': False, 'error': '限制价开关必须为明确的布尔值'} if key not in editable and key not in RUNTIME_EXTRA and not key.startswith("PMS_SECTOR_CAP_"): return {"ok": False, "error": f"参数 {key} 不可修改 (基础设施参数只在 .env 维护)"} t = _type_of(key) @@ -531,6 +555,14 @@ def set_param(key: str, value, updated_by: str = "user") -> dict: _RANGES = { + 'PMS_LIMIT_FETCH_TIMEOUT_SEC': (1, 10), + 'PMS_LIMIT_ROUND_BUDGET_SEC': (5, 60), + 'PMS_LIMIT_ROUND_MAX_CODES': (1, 60), + 'PMS_LIMIT_MIN_REQUEST_INTERVAL_SEC': (1, 10), + 'PMS_LIMIT_RETRY_BASE_SEC': (30, 600), + 'PMS_LIMIT_RETRY_MAX_SEC': (60, 3600), + 'PMS_LIMIT_MAX_FAILURES_PER_DAY': (1, 100), + 'PMS_LIMIT_REFRESH_SEC': (300, 7200), 'PMS_EXEC_REJECT_COOLDOWN_SEC': (1, 3600), 'PMS_EXEC_REJECT_MAX_COOLDOWN_SEC': (1, 86400), 'PMS_EXEC_BASIS_REFRESH_SEC': (0, 3600), diff --git a/app/services/price_limit_collector.py b/app/services/price_limit_collector.py new file mode 100644 index 0000000..c4cf84e --- /dev/null +++ b/app/services/price_limit_collector.py @@ -0,0 +1,250 @@ +"""155模拟实例的独立限制价采集;不调用交易、研判或QMT动作。""" +import copy +import re +import time +import uuid +from datetime import datetime, timedelta + +from config.settings import settings +from app.core import price_limits as core, tradedays +from app.core.execution_policy import local_now, timestamp +from app.db import session as db +from app.repo import rotation_repo as store +from app.services import param_store, price_limit_targets, price_limit_provider +from app.services.price_limits import day_key + +LEASE_KEY = 'price-limits:collector' +LEASE_SECONDS = 300 +TARGET_CAP = 1200 + + +def _summary(status, day, targets=(), entries=None, errors=(), requests=0): + entries = entries or {} + errors = list(errors) + [c+': '+entries[c]['last_error'] for c in targets + if entries.get(c, {}).get('last_error')] + valid = sum(entries.get(c, {}).get('status') == 'VALID' for c in targets) + return {'status': status, 'day': day, 'target_count': len(targets), + 'valid_count': valid, 'missing_count': len(targets)-valid, + 'error_count': len(errors), 'errors': list(errors), 'requests': requests} + + +def _acquire(owner, now): + with store.locked(LEASE_KEY, 'price_limit_lease', {}) as lease: + until = timestamp(lease.get('until')) + if until and until > now: + return False + lease.update(owner=owner, until=(now+timedelta(seconds=LEASE_SECONDS)).isoformat()) + return True + + +def _release(owner): + with store.locked(LEASE_KEY, 'price_limit_lease', {}) as lease: + if lease.get('owner') == owner: + lease.pop('owner', None) + lease.pop('until', None) + + +def _save(owner, document, now): + # 租约锁与缓存锁同一事务;不把HTTP或目标读取放进事务。 + with store.locked(LEASE_KEY, 'price_limit_lease', {}) as lease: + until = timestamp(lease.get('until')) + if lease.get('owner') != owner or not until or now >= until: + return False + with store.locked(day_key(document['trade_date']), 'price_limits', {}) as value: + value.clear() + value.update(document) + return True + + +def _config(): + p = param_store + return {'timeout': p.get_int('PMS_LIMIT_FETCH_TIMEOUT_SEC', 5), + 'budget': p.get_int('PMS_LIMIT_ROUND_BUDGET_SEC', 30), + 'max_codes': p.get_int('PMS_LIMIT_ROUND_MAX_CODES', 25), + 'interval': p.get_float('PMS_LIMIT_MIN_REQUEST_INTERVAL_SEC', 1.2), + 'retry_base': p.get_int('PMS_LIMIT_RETRY_BASE_SEC', 60), + 'retry_max': p.get_int('PMS_LIMIT_RETRY_MAX_SEC', 900), + 'max_failures': p.get_int('PMS_LIMIT_MAX_FAILURES_PER_DAY', 32), + 'refresh': p.get_int('PMS_LIMIT_REFRESH_SEC', 900)} + + +def _failure(entry, reason, now, cfg, quarantine=False): + # 连接故障保留此前有效内容与原获取时间;确切坏值才撤销可执行资格。 + entry['failures'] = min(100, int(entry.get('failures') or 0)+1) + entry['last_attempt'] = now.isoformat() + entry['last_error'] = reason + wait = min(cfg['retry_max'], cfg['retry_base'] * (2 ** min(entry['failures']-1, 10))) + entry['next_try'] = (now+timedelta(seconds=wait)).isoformat() + if quarantine or entry.get('status') != 'VALID': + entry['status'] = 'INVALID' if quarantine else 'MISSING' + + +def _accept(entry, record, source, now, cfg): + old = entry.get('record') or {} + if old and old.get('version') != record['version']: + history = entry.setdefault('revisions', []) + history.append({'record': old, 'source': entry.get('source'), + 'fetched_at': entry.get('fetched_at'), 'superseded_at': now.isoformat()}) + entry['revisions'] = history[-5:] + entry.update(status='VALID', record=record, source=source, fetched_at=now.isoformat(), + last_attempt=now.isoformat(), last_error='', failures=0, + next_try=(now+timedelta(seconds=cfg['refresh'])).isoformat()) + + +def _shared_rows(day, codes): + if not codes: + return [] + params = {'day': day} + params.update({f'c{i}': c for i, c in enumerate(codes)}) + names = ','.join(':'+k for k in params if k != 'day') + return db.fetch_all('SELECT ts_code,trade_date,pre_close,up_limit,down_limit FROM gp_stk_limit ' + 'WHERE trade_date=:day AND ts_code IN ('+names+')', params) + + +def _reserve_request(owner, now, interval): + """持久化限速水位;重建进程也不会立刻重复请求。""" + with store.locked(LEASE_KEY, 'price_limit_lease', {}) as lease: + until = timestamp(lease.get('until')) + if lease.get('owner') != owner or not until or now >= until: + return None + last = timestamp(lease.get('next_request_at')) + if last and last > now: + return (last-now).total_seconds() + lease['next_request_at'] = (now+timedelta(seconds=interval)).isoformat() + return 0.0 + + +def run_once(now=None): + """只采集实际北京日;now仅供隔离测试注入时钟,无历史补数入口。""" + current = local_now(now) + day = current.date().isoformat() + if settings.PMS_TABLE_PREFIX: + return _summary('UNAUTHORIZED_INSTANCE', day) + if not param_store.get_bool('PMS_LIMIT_COLLECT_ENABLED', False): + return _summary('DISABLED', day) + if tradedays.calendar_degraded(current.date()) or not tradedays.is_trade_day(current.date()): + return _summary('CALENDAR_WAIT', day) + try: + start = datetime.strptime(param_store.get('PMS_LIMIT_COLLECT_START', '08:55'), '%H:%M').time() + end = datetime.strptime(param_store.get('PMS_LIMIT_COLLECT_END', '18:00'), '%H:%M').time() + except (TypeError, ValueError): + return _summary('INVALID_WINDOW', day) + if not start <= current.time().replace(tzinfo=None) <= end: + return _summary('OUTSIDE_WINDOW', day) + owner = uuid.uuid4().hex + if not _acquire(owner, current): + return _summary('BUSY', day) + started = time.monotonic() + # 实际墙钟由单调时钟增量推进,测试也能准确覆盖跨日、租约和超时。 + clock = lambda: current+timedelta(seconds=time.monotonic()-started) + try: + cfg = _config() + if not (1 <= cfg['timeout'] <= 10 and 5 <= cfg['budget'] <= 60 and + 1 <= cfg['max_codes'] <= 60 and 1 <= cfg['interval'] <= 10 and + 30 <= cfg['retry_base'] <= 600 and cfg['retry_base'] <= cfg['retry_max'] <= 3600 and + 1 <= cfg['max_failures'] <= 100 and 300 <= cfg['refresh'] <= 7200): + return _summary('INVALID_CONFIG', day) + target_info = price_limit_targets.collect_targets(day) + all_codes = sorted({c for c in target_info['codes'] if core.supported(c)}) + codes, errors = all_codes[:TARGET_CAP], list(target_info['errors']) + if len(all_codes) > TARGET_CAP: + errors.append('采集目标超过1200只,未覆盖部分保持等待') + previous, _ = store.get(day_key(day)) + document = copy.deepcopy(previous) if isinstance(previous, dict) and previous.get('trade_date') == day else {} + document.update(trade_date=day, schema_version=1, targets=codes, + target_sources={c: target_info['sources'].get(c, []) for c in codes}) + entries = document.get('entries') + if not isinstance(entries, dict): + entries = document['entries'] = {} + # 再校验已存值,损坏或异日内容不能靠VALID标记继续使用。 + for code, entry in list(entries.items()): + if not core.supported(code) or not isinstance(entry, dict): + entries.pop(code, None) + continue + if type(entry.get('failures')) is not int or not 0 <= entry['failures'] <= 100: + entry['failures'] = 0 + for field in ('next_try', 'last_attempt'): + stamp = timestamp(entry.get(field)) + if not stamp or stamp.date().isoformat() != day: + entry.pop(field, None) + else: + entry[field] = stamp.isoformat() + if not isinstance(entry.get('revisions'), list): + entry['revisions'] = [] + for field in ('last_error', 'shared_invalid_reason'): + reason = entry.get(field) + if reason and (not isinstance(reason, str) or not re.fullmatch('[A-Z_]{1,64}', reason)): + entry[field] = 'CACHE_META_INVALID' + if entry.get('status') == 'VALID': + checked = core.normalize_rows([entry.get('record')], day, [code])['records'].get(code) + if (not checked or checked['version'] != (entry.get('record') or {}).get('version') + or entry.get('source') not in ('gp_stk_limit', 'tushare.stk_limit')): + entry.update(status='INVALID', last_error='CACHE_INVALID', next_try=current.isoformat()) + for code in codes: + entries.setdefault(code, {'status': 'MISSING', 'failures': 0}) + try: + shared = core.normalize_rows(_shared_rows(day, codes), day, codes) if codes else {'records': {}, 'invalid': {}, 'errors': []} + errors.extend(shared['errors']) + for code, record in shared['records'].items(): + entry = entries[code] + if entry.get('source') == 'tushare.stk_limit': + continue + if entry.get('status') != 'VALID' or (entry.get('record') or {}).get('version') != record['version']: + _accept(entry, record, 'gp_stk_limit', clock(), cfg) + entry.pop('shared_invalid_reason', None) + for code, reason in shared['invalid'].items(): + if code in entries and entries[code].get('source') != 'tushare.stk_limit': + entry = entries[code] + # 相同坏值不在每轮重置退避;第一次仍允许直接向Tushare补齐。 + if entry.get('shared_invalid_reason') != reason: + entry.update(status='INVALID', last_error=reason, next_try=current.isoformat()) + entry['shared_invalid_reason'] = reason + except Exception: + errors.append('共享限制价读取失败') + # 没有采到过的票先查,避免前排失败长时间挤占后排。每票退避独立。 + due = [c for c in codes if int(entries[c].get('failures') or 0) < cfg['max_failures'] + and (timestamp(entries[c].get('next_try')) or current) <= clock()] + due.sort(key=lambda c: (entries[c].get('status') == 'VALID', entries[c].get('last_attempt') or '', c)) + deadline = time.monotonic()+cfg['budget'] + count = 0 + if not settings.PMS_TUSHARE_TOKEN and due: + errors.append('采集密钥未配置') + else: + for code in due: + left = deadline-time.monotonic() + if count >= cfg['max_codes'] or left < 1 or clock().date().isoformat() != day: + break + delay = _reserve_request(owner, clock(), cfg['interval']) + if delay is None: + return _summary('LEASE_LOST', day, codes, entries, errors, count) + if delay: + if delay+1 > left: + break + time.sleep(delay) + delay = _reserve_request(owner, clock(), cfg['interval']) + if delay is None or delay > 0: + break + result = price_limit_provider.fetch_limit(code, day, settings.PMS_TUSHARE_TOKEN, + timeout_sec=min(cfg['timeout'], max(1, deadline-time.monotonic()))) + count += 1 + at = clock() + if at.date().isoformat() != day or time.monotonic() > deadline: + _failure(entries[code], 'ROUND_TIMEOUT', at, cfg) + errors.append('本轮采集超时,迟到响应未采纳') + break + entry = entries[code] + if code in result['records']: + _accept(entry, result['records'][code], 'tushare.stk_limit', at, cfg) + else: + invalid = result.get('invalid') or {} + reason = invalid.get(code) or (result['errors'][0] if result['errors'] else ('UNEXPECTED_SECURITY' if invalid else 'NOT_PUBLISHED')) + _failure(entry, reason, at, cfg, quarantine=bool(invalid)) + end_at = clock() + summary = _summary('COMPLETE' if not errors and all(entries[c].get('status') == 'VALID' and not entries[c].get('last_error') for c in codes) else 'PARTIAL', + day, codes, entries, errors, count) + document.update(last_round_at=end_at.isoformat(), last_summary=summary) + if end_at.date().isoformat() != day or not _save(owner, document, end_at): + return _summary('LEASE_LOST', day, codes, entries, errors, count) + return summary + finally: + _release(owner) diff --git a/app/services/price_limit_provider.py b/app/services/price_limit_provider.py new file mode 100644 index 0000000..ab2df42 --- /dev/null +++ b/app/services/price_limit_provider.py @@ -0,0 +1,109 @@ +"""单证券限制价主源。只访问固定地址,不重试、不输出凭据或响应正文。 + +Timeout.total 限定一次 HTTP 连接/读取的超时预算;流式读取还会检查单调时钟。 +DNS 解析及两次流块之间的阻塞不保证被总预算中断,因此这不是绝对墙钟期限。 +整轮预算、重试和退避由采集任务负责,执行报价路径只读本地缓存。 +""" +import json +import math +import time + +import requests +from urllib3.exceptions import TimeoutError as TransportTimeout +from urllib3.util import Timeout + +from app.core import price_limits + +ENDPOINT = 'https://api.tushare.pro' +FIELDS = 'ts_code,trade_date,pre_close,up_limit,down_limit' +MAX_BODY_BYTES = 64 * 1024 + + +def _error(reason): + return {'records': {}, 'invalid': {}, 'errors': [reason]} + + +def fetch_limit(code, day, token, timeout_sec=5): + """查询指定交易日的一只沪深证券;所有失败仅返回固定原因码。""" + if not price_limits.supported(code): + return _error('INVALID_REQUEST_CODE') + code = code.strip().upper() + try: + target = price_limits._day(day) + except (TypeError, ValueError): + return _error('INVALID_TARGET_DATE') + if not isinstance(token, str) or not token.strip(): + return _error('MISSING_PROVIDER_TOKEN') + if isinstance(timeout_sec, bool): + return _error('INVALID_HTTP_TIMEOUT') + try: + budget = float(timeout_sec) + except (TypeError, ValueError, OverflowError): + return _error('INVALID_HTTP_TIMEOUT') + if not math.isfinite(budget) or budget <= 0: + return _error('INVALID_HTTP_TIMEOUT') + + session = response = None + deadline = time.monotonic() + budget + try: + session = requests.Session() + # 不读取环境代理、.netrc 或环境 CA 设置,凭据只发往固定 HTTPS 地址。 + session.trust_env = False + response = session.post(ENDPOINT, json={ + 'api_name': 'stk_limit', 'token': token.strip(), + 'params': {'ts_code': code, 'trade_date': target.replace('-', '')}, + 'fields': FIELDS, + }, headers={'Accept': 'application/json', 'Accept-Encoding': 'identity'}, + allow_redirects=False, stream=True, + timeout=Timeout(total=budget, connect=min(2., budget), read=budget)) + if time.monotonic() >= deadline: + return _error('HTTP_TIMEOUT') + if response.status_code != 200: + return _error('HTTP_ERROR') + declared = response.headers.get('Content-Length') + if declared is not None: + try: + length = int(declared) + except (TypeError, ValueError, OverflowError): + return _error('INVALID_PROVIDER_RESPONSE') + if length < 0: + return _error('INVALID_PROVIDER_RESPONSE') + if length > MAX_BODY_BYTES: + return _error('RESPONSE_TOO_LARGE') + body = bytearray() + for chunk in response.iter_content(chunk_size=4096): + if time.monotonic() >= deadline: + return _error('HTTP_TIMEOUT') + if not isinstance(chunk, bytes): + return _error('INVALID_PROVIDER_RESPONSE') + if len(body) + len(chunk) > MAX_BODY_BYTES: + return _error('RESPONSE_TOO_LARGE') + body.extend(chunk) + if time.monotonic() >= deadline: + return _error('HTTP_TIMEOUT') + try: + decoded = json.loads(body) + # 永远传非空单代码白名单,不能借空集合放开到全市场。 + result = price_limits.provider_decode(decoded, target, requested_codes=[code]) + except Exception: + return _error('INVALID_PROVIDER_RESPONSE') + if time.monotonic() >= deadline: + return _error('HTTP_TIMEOUT') + return result + except (requests.exceptions.Timeout, TransportTimeout): + return _error('HTTP_TIMEOUT') + except requests.exceptions.ConnectionError as exc: + # requests 的流式读取会把 urllib3 读超时包装成 ConnectionError。 + if any(isinstance(cause, TransportTimeout) for cause in exc.args): + return _error('HTTP_TIMEOUT') + return _error('HTTP_ERROR') + except Exception: + # Requests 异常可包含URL、请求信息或上游正文,禁止回显和记完整日志。 + return _error('HTTP_ERROR') + finally: + for resource in (response, session): + if resource is not None: + try: + resource.close() + except Exception: + pass # 清理异常不覆盖固定错误结果,也不输出潜在凭据。 diff --git a/app/services/price_limit_targets.py b/app/services/price_limit_targets.py new file mode 100644 index 0000000..a4a3593 --- /dev/null +++ b/app/services/price_limit_targets.py @@ -0,0 +1,132 @@ +"""独立采集进程的限制价目标清单。只读资料,不授权交易或写入业务状态。 + +监控读取复用 rotation_service.pull,HTTP 连接和读取超时分别为 3、15 秒。 +本函数不得在下单事务或账户锁内调用。未冻结快照只用于预取限制价, +实际轮换仍须执行原有的冻结、信号、资金和权限校验。 +""" +import datetime as dt +import re + +from app.core import micro_rotation, strategy_book as books +from app.repo import pms_repo, qmt_repo, rotation_repo +from app.services import rotation_service + + +SOURCE_CAP = 5000 +# 仓储 recent 自身最多返回 1000 行,不能按 5000 行误判为读取完整。 +SNAPSHOT_CAP = 1000 +LIVE = ('PROPOSED', 'RULE_PASSED', 'JUDGE_PASSED', 'DISPATCHED', 'WAIT_REVIEW') + + +def collect_targets(day): + """返回证券、逐来源错误和来源名;一个来源失败不丢弃其他已知证券。""" + day = day.isoformat() if isinstance(day, dt.date) and not isinstance(day, dt.datetime) else day + try: + if not isinstance(day, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}', day): + raise ValueError('date') + dt.date.fromisoformat(day) + except (TypeError, ValueError): + return {'codes': [], 'errors': ['目标交易日格式无效'], 'sources': {}} + + sources, errors = {}, [] + + def error(reason): + if reason not in errors: + errors.append(reason) + + def add(code, name): + code = code.strip().upper() if isinstance(code, str) else '' + if micro_rotation.mainland_code(code): + sources.setdefault(code, set()).add(name) + + def read(name, loader, cap=SOURCE_CAP): + try: + rows = loader() + except Exception: + error(name+'读取失败') + return [] + if not isinstance(rows, list): + error(name+'记录格式异常') + return [] + if len(rows) >= cap: + error(name+'达到读取上限,结果可能不完整') + return rows[:cap] + + with books.scope('all'): + loaders = ( + ('策略持仓', lambda: pms_repo.list_positions(only_open=True)), + ('待执行指令', lambda: pms_repo.list_instructions(statuses=LIVE, limit=SOURCE_CAP)), + ('QMT在途委托', lambda: qmt_repo.list_orders(statuses=qmt_repo.LIVE, limit=SOURCE_CAP)), + ('执行计划', lambda: pms_repo.list_plans(statuses=('PENDING', 'GATED', 'EXECUTING'), limit=SOURCE_CAP)), + ('交易建议', lambda: pms_repo.list_proposals(statuses=('WAIT_USER', 'ACCEPTED'), limit=SOURCE_CAP)), + ) + for name, loader in loaders: + for row in read(name, loader): + if not isinstance(row, dict) or 'ts_code' not in row: + error(name+'记录格式异常') + continue + add(row['ts_code'], name) + + try: + state, _ = rotation_repo.get('state:live') + rotation = (state or {}).get('rotation') or {} + targets = rotation.get('target') or [] + if not isinstance(targets, list): + raise ValueError('target') + if len(targets) >= SOURCE_CAP: + error('微盘执行目标达到读取上限,结果可能不完整') + for code in targets[:SOURCE_CAP]: + add(code, '微盘执行目标') + except Exception: + error('微盘执行目标读取失败') + + try: + config = rotation_service.configuration() + rule = config['rule_version'] + pool_size = config.get('candidate_pool_size', 100) + micro_rotation.rule_parameters(rule, pool_size) + except Exception: + error('微盘运行规则读取失败') + config = None + + def snapshot_codes(value, name): + if not isinstance(value, dict): + error(name+'记录格式异常') + return + # effective_date 是既有快照的执行日字段;历史日期和历史规则不参与本次采集。 + if value.get('effective_date') != day or value.get('rule_version') != rule: + return + try: + issues = micro_rotation.snapshot_check(value, day, require_frozen=False, + expected_rule=rule, candidate_pool_size=pool_size) + except Exception: + issues = ['invalid'] + if issues: + error(name+'校验未通过') + return + if not value.get('basket_ready'): + error(name+'名单尚未就绪') + return + members = value['candidates'] if rule == micro_rotation.EXECUTION_RULE else value['members'] + for member in members: + add(member['code'], name) + + if config is not None: + for row in read('近期微盘快照', lambda: rotation_repo.recent('snapshot', limit=SNAPSHOT_CAP), SNAPSHOT_CAP): + snapshot_codes(row.get('value') if isinstance(row, dict) else None, '微盘保存快照') + if not config.get('url'): + error('微盘监控地址未配置') + else: + try: + value = rotation_service.pull(day) + except Exception: + error('微盘监控快照读取失败') + else: + if not isinstance(value, dict) or value.get('effective_date') != day or value.get('rule_version') != rule: + error('微盘监控快照日期或规则不符') + else: + snapshot_codes(value, '微盘监控快照') + + codes = sorted(sources) + return {'codes': codes, 'errors': errors, + 'sources': {code: sorted(sources[code]) for code in codes}} diff --git a/app/services/price_limits.py b/app/services/price_limits.py new file mode 100644 index 0000000..5877b90 --- /dev/null +++ b/app/services/price_limits.py @@ -0,0 +1,62 @@ +"""执行限制价的本地只读入口,不调用外部行情接口。""" +from app.core import price_limits as core +from app.core.execution_policy import timestamp, local_now +from app.db import session as db +from app.repo import rotation_repo as store +from app.services import param_store + +SOURCES = ('gp_stk_limit', 'tushare.stk_limit') + + +def day_key(day): + return 'price-limits:' + core._day(day) + + +def read_cache(day): + """坏缓存按无值处理;执行读取不更新缓存、不延长有效日期。""" + target = core._day(day) + value, _ = store.get(day_key(target)) + if not isinstance(value, dict) or value.get('trade_date') != target: + return {} + entries = value.get('entries') + if not isinstance(entries, dict): + return {} + valid = {} + for code, entry in entries.items(): + if not core.supported(code) or not isinstance(entry, dict): + continue + record = entry.get('record') + if entry.get('status') != 'VALID' or entry.get('source') not in SOURCES or not entry.get('fetched_at'): + continue + fetched = timestamp(entry['fetched_at']) + if not fetched or fetched.date().isoformat() != target or fetched > local_now(): + continue + result = core.normalize_rows([record], target, [code]) + normalized = result['records'].get(code) + if normalized and normalized['version'] == (record or {}).get('version'): + valid[code] = dict(normalized, source=entry['source'], fetched_at=entry['fetched_at']) + return valid + + +def get_limits(day, codes=None): + """开关开启后只读本实例当日缓存;失败不混用其他日期或推算来源。""" + day = core._day(day) + wanted = None if codes is None else {str(c).strip().upper() for c in codes if core.supported(c)} + if wanted == set(): + return {} + if param_store.get_bool('PMS_LIMIT_CACHE_ENABLED', False): + try: + result = read_cache(day) + except Exception: + return {} + return {c: r for c, r in result.items() if wanted is None or c in wanted} + # 关闭新功能时保留原有数据路径,但在读取入口限定沪深证券。 + params = {'day': day} + where = '' + if wanted is not None: + params.update({f'c{i}': c for i, c in enumerate(sorted(wanted))}) + where = ' AND ts_code IN (' + ','.join(':'+k for k in params if k != 'day') + ')' + else: + where = " AND (ts_code LIKE '60____.SH' OR ts_code LIKE '68____.SH' OR ts_code LIKE '00____.SZ' OR ts_code LIKE '30____.SZ')" + rows = db.fetch_all('SELECT ts_code,up_limit,down_limit FROM gp_stk_limit WHERE trade_date=:day'+where, params) + return {r['ts_code']: r for r in rows if core.supported(r.get('ts_code'))} diff --git a/app/services/rotation_service.py b/app/services/rotation_service.py index 941134d..35e94fc 100644 --- a/app/services/rotation_service.py +++ b/app/services/rotation_service.py @@ -253,8 +253,12 @@ def _quote(code, now): def _limits(day): + if param_store.get_bool('PMS_LIMIT_CACHE_ENABLED', False): + from app.services import price_limits + return price_limits.get_limits(day) from app.db import session as db - rows = db.fetch_all('SELECT ts_code,up_limit,down_limit FROM gp_stk_limit WHERE trade_date=:day', {'day': day}) + rows = db.fetch_all('SELECT ts_code,up_limit,down_limit FROM gp_stk_limit WHERE trade_date=:day ' + "AND (ts_code LIKE '60____.SH' OR ts_code LIKE '68____.SH' OR ts_code LIKE '00____.SZ' OR ts_code LIKE '30____.SZ')", {'day': day}) return {r['ts_code']: r for r in rows} diff --git a/config/settings.py b/config/settings.py index ba83629..cca10e9 100644 --- a/config/settings.py +++ b/config/settings.py @@ -17,6 +17,22 @@ from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + # 当日限制价缓存与独立采集。业务开关和预算统一从ParamStore读取。 + PMS_LIMIT_CACHE_ENABLED: bool = False + PMS_LIMIT_COLLECT_ENABLED: bool = False + PMS_LIMIT_FETCH_TIMEOUT_SEC: int = 5 + PMS_LIMIT_ROUND_BUDGET_SEC: int = 30 + PMS_LIMIT_ROUND_MAX_CODES: int = 25 + PMS_LIMIT_MIN_REQUEST_INTERVAL_SEC: float = 1.2 + PMS_LIMIT_RETRY_BASE_SEC: int = 60 + PMS_LIMIT_RETRY_MAX_SEC: int = 900 + PMS_LIMIT_MAX_FAILURES_PER_DAY: int = 32 + PMS_LIMIT_REFRESH_SEC: int = 900 + PMS_LIMIT_COLLECT_START: str = "08:55" + PMS_LIMIT_COLLECT_END: str = "18:00" + # 密钥只由采集容器专用.env.market-data注入,不得写入共享.env或ParamStore。 + PMS_TUSHARE_TOKEN: str = "" + # 异步研判与委托报价;业务运行值统一从ParamStore读取。 PMS_EXEC_REJECT_COOLDOWN_SEC: int = 60 PMS_EXEC_REJECT_MAX_COOLDOWN_SEC: int = 900 diff --git a/docker-compose.yml b/docker-compose.yml index cf07838..369007b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,10 +9,11 @@ # PMS_WEB_PORT_HOST=38200 → 页面宿主机端口 (容器内固定 38100, 不用改) # COMPOSE_PROJECT_NAME=pms-real → compose 项目名 (网络/卷前缀), docker 原生支持 # 这些键进容器环境也无害: settings 是 extra="ignore", 不声明就当不存在。 -# 三组服务: +# 四组服务: # 默认 pms-web 管理页面 (38100) # --profile sched pms-beat / pms-worker Celery 调度与执行 # --profile ws pms-ws QMT WebSocket 常驻连接 (协议 V1.0) +# --profile market-data pms-market-data 独立限制价采集,不接交易通道 # # pms-ws 与 worker 的关系: worker 里的 executor 把待发委托写进 pms_qmt_order 出口表, # pms-ws 亚秒轮询取走并发出。两者**只经数据库耦合**, 谁先起谁后起都不影响 —— pms-ws 没起 @@ -68,6 +69,27 @@ services: profiles: ["sched"] command: celery -A app.scheduler.celery_app worker -l info -c 2 + # 独立单进程采集,专用密钥不注入其他服务。不要扩副本或同时启动第二个采集入口。 + pms-market-data: + <<: *pms-base + container_name: ${PMS_STACK_PREFIX:-}pms-market-data + profiles: ["market-data"] + env_file: + - .env + - path: .env.market-data + required: false + command: python -m app.market_data_worker + volumes: [] + cpus: 0.5 + mem_limit: 256m + read_only: true + tmpfs: + - /tmp:size=16m,mode=1777 + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + stop_grace_period: 60s + stop_signal: SIGTERM + # QMT WebSocket 常驻连接。协议 §1 规定 PMS 只开一条连接 (多开会让指令乱序), # 所以这个服务**绝不能扩副本** —— 不要 deploy.replicas, 不要 docker compose up --scale。 # 退出流程见 app/ws/runner.py 的 _shutdown: 刷水位 → 最后一次 ack → 关连接 → 清心跳, diff --git a/docs/开发入口.md b/docs/开发入口.md index 6e87bd6..991988f 100644 --- a/docs/开发入口.md +++ b/docs/开发入口.md @@ -13,7 +13,7 @@ | 宏观价量网格、同步形成与日线趋势 | [宏观网格研究接续](宏观网格研究接续_2026-10-29.md),含三因素增量、八幅研究图、各风格与数据末日区间判断 | | 筹码断档、套牢盘与吸筹次数(已结束) | [研究归档](筹码断档与吸筹研究接续_2026-10-30.md),用户决定不再继续;历史数据口径不代表现行研究范围 | | PMS通用迁移与既有修复 | [开发接续](开发接续_2026-10-12.md),按主题查对应节;旧时点未完成项可能已有后续结论 | -| 盘中改价、异步研判与微盘递补发布 | [执行链路发布接续](执行报价与异步研判发布接续_2026-11-03.md),区分部署、开关与自然判收;[实施规格](执行报价与异步研判实施规格_2026-11-03.md) | +| 盘中改价、异步研判与微盘递补发布 | [执行链路发布接续](执行报价与异步研判发布接续_2026-11-03.md),区分部署、开关与自然判收;[实施规格](执行报价与异步研判实施规格_2026-11-03.md);[当日限制价采集边界](当日限制价采集边界_2026-11-03.md) | | 架构与接口 | [README模块地图](../README.md)、[总体设计](../POSITION_MGMT_DESIGN.md);改接口再读[QMT协议](../QMT_WS_PROTOCOL.md)、[bionic接口](../BIONIC_PMS_INTERFACE.md)或[计划接口](../UPSTREAM_PLAN_API.md) | | 后端、前端、测试、研究 | [后端规则](../app/AGENTS.md)、[页面规则](../app/web/AGENTS.md)、[脚本规则](../scripts/AGENTS.md)、[研究规则](../research/AGENTS.md),仅选择所涉及目录 | | 部署与运行核验 | [pms-release](../.agents/skills/pms-release/SKILL.md);命令目录为`make help`,并非所有目标都是只读 | diff --git a/docs/当日限制价采集边界_2026-11-03.md b/docs/当日限制价采集边界_2026-11-03.md new file mode 100644 index 0000000..95171ba --- /dev/null +++ b/docs/当日限制价采集边界_2026-11-03.md @@ -0,0 +1,29 @@ +# 当日限制价采集边界 + +本方案在2026年11月3日经用户同意实施,只部署155模拟PMS。盘中改价、微盘v3和异步研判已开启;此次补齐执行所需的当日价格边界,不修改交易授权、数量或策略资格。 + +## 数据来源和作用范围 + +独立采集进程先读取目标股票在gp_stk_limit的当日记录,再用Tushare stk_limit补齐缺失值。2026年11月3日实测HTTPS单票查询有当日数据,逗号连接两只证券会返回空,因此只按单票请求,并统一限速。只收集沪深A股的实际持仓、未完成指令、在途委托、待执行计划、待确认或已采纳建议、当前微盘目标及当前规则的完整候选池;入口排除北交所。 + +日线计算只作为后续辅助核对的研究方向,本次不产生可下单的推算限制价,不使用复权昨收、昨日涨跌停价或大模型价格区间替代正式当日数据。缺限制价不是停牌,也不成为微盘递补理由。 + +## 校验与缓存 + +每条记录绑定证券、交易日期、原始昨收、涨停价、跌停价、内容摘要、来源及实际获取时间。检查日期一致、价格有限且为有效分位、上下限关系、重复冲突和来源范围。零上下限或不支持的无限价情形保留等待。结构异常拒绝该次响应;同票异常隔离该票,不污染其他证券。 + +数据保存到本实例既有pms_rotation_document的price_limits文档,按交易日隔离。采集只写该自有缓存及采集租约,不改共享gp_stk_limit,不改账本、队列、快照冻结时间或委托。执行链只读取本地缓存,绝不临时请求Tushare。跨日自然失效;同日连接故障保留此前有效记录及原获取时间,有明确坏值或冲突证据则隔离该票。来源发生修订保留前一摘要与修订时间。 + +## 及时性和故障边界 + +新增独立pms-market-data进程,不使用Celery交易worker或模型队列,不建立QMT连接。默认交易日08:55至18:00运行,日历不可确认时不采集。每轮HTTP预算30秒、最多25只,单次5秒、两请求至少1.2秒;每10秒重新检查。缺失或失败逐票按60秒起、最多900秒退避,每票每日连续失败最多32次;有效记录每900秒复核。业务参数统一通过ParamStore读取。 + +单次网络超时与流式截止共同限制等待,但操作系统DNS等底层阻塞不能声称绝对墙钟保证。进程独立和有限租约保证此类异常不会占用交易任务;采集返回后再检查租约与截止,过期执行者不能覆盖后来结果。各来源读取异常明确记录,已有其他目标继续采集;达到读取上限报告不完整,不能伪称全覆盖。 + +一只缺数据只让该票WAIT,其他证券继续。若整个微盘篮子都没有可执行买入条件,沿用既有“不先卖原仓筹资”的保护。停止采集或回退时保留已有缓存、账本与回报;缓存开关关闭退回原来的当日gp_stk_limit读取,不会开启推算兜底。 + +## 密钥与发布 + +Tushare密钥只注入独立采集容器的私有.env.market-data,登记SECRET_KEYS;不进入主.env、ParamStore、页面、日志、镜像或Git。使用固定HTTPS地址,关闭重定向和环境代理,不输出响应正文或上游错误消息。155专用环境文件另外为采集进程的数据库连接设置5秒读写超时,不更改其他服务连接配置。专用文件缺失时不影响默认服务编排;采集无密钥会明确报告缺配置。 + +先完成开发机离线测试和候选镜像断网全量回归,再按实际消费关系定向重建155服务并启动采集。先检查当日目标覆盖,再开启本地缓存读取;数据未齐明确记录,不用虚假数据冒充判收。明日一次性核验自然08:50冻结、当日候选限制价、真实异步请求与超时结果;没有自然事件就保留未判收。 diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 1241c4c..c5f117c 100644 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -6,6 +6,9 @@ test_batch54_units.py 用户候选来源、到期核验、权限、显式取用与技术面补读 (9例) +新增限制价采集专项:core 25例、provider 21例、targets 18例、collector 28例, +缓存消费23例、独立进程入口6例。 + 包含 (例数按 2026-10-26 隔离回归实跑校正): test_batch57_units.py 异步请求持久化、截止时间、旧结果隔离、失败续查、真实调度入口及行情来源 (24例) test_batch56_units.py 微盘v3冻结候选、按序递补与部分成交保护 (40例) @@ -189,7 +192,10 @@ import sys HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) -SUITES = ["test_batch54_units.py", +SUITES = ["test_price_limits_core.py", "test_price_limit_provider.py", + "test_price_limit_targets.py", "test_price_limit_collector.py", + "test_price_limit_cache.py", "test_market_data_worker.py", + "test_batch54_units.py", "test_sim_reset_units.py", "test_batch53_units.py", "test_batch52_units.py", diff --git a/scripts/test_market_data_worker.py b/scripts/test_market_data_worker.py new file mode 100644 index 0000000..4c664aa --- /dev/null +++ b/scripts/test_market_data_worker.py @@ -0,0 +1,99 @@ +import unit_test_guard +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import signal +import types +import unittest +from unittest.mock import patch, Mock +from app import market_data_worker as worker +from app import services +from app.services import param_store +from config.settings import settings + + +class WorkerChecks(unittest.TestCase): + def test_invalid_limit_flags_fail_closed_without_database_write(self): + rows = {key: {'param_value': 'invalid'} for key in param_store.LIMIT_FLAGS} + with patch.object(param_store, 'refresh', return_value=rows), \ + patch.object(param_store.pms_repo, 'set_param') as write: + self.assertTrue(param_store.get_bool('PMS_LIMIT_CACHE_ENABLED')) + self.assertFalse(param_store.get_bool('PMS_LIMIT_COLLECT_ENABLED')) + self.assertFalse(param_store.set_param('PMS_LIMIT_CACHE_ENABLED', 'invalid')['ok']) + write.assert_not_called() + + def test_shutdown_finishes_current_round_and_does_not_run_again(self): + callbacks = {} + event = Mock() + stopped = [False] + event.is_set.side_effect = lambda: stopped[0] + event.set.side_effect = lambda: stopped.__setitem__(0, True) + module = types.ModuleType('app.services.price_limit_collector') + def run(): + callbacks[signal.SIGTERM](signal.SIGTERM, None) + return {'status':'done','day':'2026-11-03','target_count':3,'valid_count':2,'missing_count':1,'error_count':0, + 'private':'must-not-log'} + module.run_once = Mock(side_effect=run) + with patch.object(services,'price_limit_collector',module,create=True), \ + patch.object(worker,'Event',return_value=event), \ + patch.object(worker.signal,'signal',side_effect=lambda name,callback: callbacks.__setitem__(name,callback)), \ + patch.object(worker.logging,'basicConfig'), self.assertLogs(worker.logger,level='INFO') as logs: + worker.main() + module.run_once.assert_called_once_with() + event.wait.assert_called_once_with(10) + self.assertIn(signal.SIGINT,callbacks) + self.assertNotIn('must-not-log',str(logs.output)) + self.assertIn('missing_count=1',str(logs.output)) + + def test_failure_logs_type_only_and_next_round_runs(self): + module = types.ModuleType('app.services.price_limit_collector') + module.run_once = Mock(side_effect=[RuntimeError('private token and address'), {'status':'done'}]) + event = Mock() + event.is_set.side_effect = [False,False,True] + with patch.object(services,'price_limit_collector',module,create=True), \ + patch.object(worker,'Event',return_value=event), \ + patch.object(worker.signal,'signal'), patch.object(worker.logging,'basicConfig'), \ + self.assertLogs(worker.logger,level='INFO') as logs: + worker.main() + self.assertEqual(module.run_once.call_count,2) + self.assertEqual(event.wait.call_count,2) + self.assertIn('RuntimeError',str(logs.output)) + self.assertNotIn('private token',str(logs.output)) + + def test_summary_rejects_unsanitized_fields(self): + value=worker._summary({'status':{'token':'secret'},'day':'secret', + 'target_count':'secret','valid_count':True,'missing_count':-1,'error_count':4}) + self.assertEqual(value,('unknown','-',0,0,0,4)) + with self.assertRaises(ValueError):worker._summary([]) + + def test_token_is_environment_only_and_absent_from_snapshot(self): + self.assertEqual(settings.PMS_TUSHARE_TOKEN,'') + self.assertIn('PMS_TUSHARE_TOKEN',param_store.SECRET_KEYS) + self.assertNotIn('PMS_TUSHARE_TOKEN',param_store.RUNTIME_EXTRA) + self.assertNotIn('PMS_TUSHARE_TOKEN',param_store._editable_keys()) + with patch.object(param_store,'refresh',return_value={}) as refresh: + self.assertEqual(param_store.get('PMS_TUSHARE_TOKEN'),'') + refresh.assert_not_called() + self.assertNotIn('PMS_TUSHARE_TOKEN',{p['key'] for p in param_store.snapshot()['params']}) + self.assertFalse(param_store.set_param('PMS_TUSHARE_TOKEN','private')['ok']) + + def test_defaults_types_ranges_and_collection_window(self): + expected={'PMS_LIMIT_CACHE_ENABLED':(False,None),'PMS_LIMIT_COLLECT_ENABLED':(False,None), + 'PMS_LIMIT_FETCH_TIMEOUT_SEC':(5,(1,10)), 'PMS_LIMIT_ROUND_BUDGET_SEC':(30,(5,60)), + 'PMS_LIMIT_ROUND_MAX_CODES':(25,(1,60)), 'PMS_LIMIT_MIN_REQUEST_INTERVAL_SEC':(1.2,(1,10)), + 'PMS_LIMIT_RETRY_BASE_SEC':(60,(30,600)), 'PMS_LIMIT_RETRY_MAX_SEC':(900,(60,3600)), + 'PMS_LIMIT_MAX_FAILURES_PER_DAY':(32,(1,100)), 'PMS_LIMIT_REFRESH_SEC':(900,(300,7200)), + 'PMS_LIMIT_COLLECT_START':('08:55',None),'PMS_LIMIT_COLLECT_END':('18:00',None)} + for key,(default,bounds) in expected.items(): + self.assertEqual(getattr(settings,key),default) + self.assertEqual(param_store.RUNTIME_EXTRA[key][:2],(default,type(default))) + if bounds: + self.assertEqual(param_store._RANGES[key],bounds) + lo,hi=bounds + self.assertIsNone(param_store._range_check(key,lo)) + self.assertIsNone(param_store._range_check(key,hi)) + self.assertIsNotNone(param_store._range_check(key,lo-1)) + self.assertIsNotNone(param_store._range_check(key,hi+1)) + + +if __name__=='__main__':unittest.main() diff --git a/scripts/test_price_limit_cache.py b/scripts/test_price_limit_cache.py new file mode 100644 index 0000000..cf536ce --- /dev/null +++ b/scripts/test_price_limit_cache.py @@ -0,0 +1,237 @@ +"""限制价缓存及普通、微盘消费入口的隔离回归。无外部访问或状态写入。""" +import unit_test_guard # noqa: F401 +import contextlib +import copy +from datetime import datetime, timedelta +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import requests +from app.core import price_limits as core, execution_policy as policy +from app.services import price_limits as cache, market, rotation_service, param_store + +DAY = '2026-11-03' +NOW = datetime(2026, 11, 3, 10, 0, tzinfo=policy.BEIJING) +A, B, BJ = '600000.SH', '000001.SZ', '920001.BJ' + + +def record(code=A, day=DAY, high='11.00'): + value = {'ts_code': code, 'trade_date': day, 'pre_close': '10.00', + 'up_limit': high, 'down_limit': '9.00'} + return core.normalize_rows([value], day)['records'][code] + + +def entry(code=A, **changes): + return {'status': 'VALID', 'source': 'tushare.stk_limit', + 'fetched_at': (NOW-timedelta(minutes=5)).isoformat(), 'record': record(code), **changes} + + +class PriceLimitCacheTests(unittest.TestCase): + def setUp(self): + self.stack = contextlib.ExitStack() + self.addCleanup(self.stack.close) + self.enabled = True + self.value = {'trade_date': DAY, 'entries': {A: entry(), B: entry(B)}} + self.stack.enter_context(patch.object(cache, 'local_now', return_value=NOW)) + self.stack.enter_context(patch.object(param_store, 'get_bool', side_effect=lambda key, default=False: + self.enabled if key == 'PMS_LIMIT_CACHE_ENABLED' else default)) + self.read = self.stack.enter_context(patch.object(cache.store, 'get', + side_effect=lambda key: (copy.deepcopy(self.value), 1))) + self.legacy = self.stack.enter_context(patch.object(cache.db, 'fetch_all', return_value=[])) + self.market_legacy = self.stack.enter_context(patch.object(market, 'fetch_all', return_value=[])) + self.bars = self.stack.enter_context(patch.object(market, '_bars', return_value=[ + {'time': NOW.isoformat(), 'close': 10.0}])) + self.forbidden = [] + for owner, name in ((requests, 'get'), (requests, 'post'), (requests, 'Session'), + (cache.db, 'execute'), (cache.db, 'fetch_one'), (cache.db, 'transaction'), + (cache.store, 'locked'), (cache.store, 'put_once'), (cache.store, 'stage_locked')): + self.forbidden.append(self.stack.enter_context(patch.object(owner, name, + side_effect=AssertionError('缓存消费禁止联网或写入')))) + + def tearDown(self): + for method in self.forbidden: + method.assert_not_called() + + def assert_no_legacy(self): + self.legacy.assert_not_called() + self.market_legacy.assert_not_called() + + def test_valid_cache_keeps_content_version_and_source_metadata(self): + result = cache.get_limits(DAY, [A]) + self.assertEqual(result[A], dict(record(), source='tushare.stk_limit', fetched_at=entry()['fetched_at'])) + self.read.assert_called_once_with('price-limits:'+DAY) + self.assert_no_legacy() + + def test_shared_source_is_allowed_only_after_same_validation(self): + self.value['entries'][A]['source'] = 'gp_stk_limit' + self.assertEqual(cache.read_cache(DAY)[A]['source'], 'gp_stk_limit') + self.value['entries'][A]['record']['version'] = 'corrupt' + self.assertNotIn(A, cache.read_cache(DAY)) + + def test_envelope_wrong_date_or_structure_returns_no_value(self): + for value in (None, [], {'trade_date': '2026-11-02', 'entries': {A: entry()}}, + {'trade_date': DAY, 'entries': []}, {'trade_date': DAY}): + with self.subTest(value_type=type(value).__name__): + self.value = value + self.assertEqual(cache.read_cache(DAY), {}) + self.assert_no_legacy() + + def test_record_wrong_date_is_rejected_even_with_its_own_valid_hash(self): + self.value['entries'][A]['record'] = record(day='2026-11-02') + self.assertEqual(set(cache.read_cache(DAY)), {B}) + + def test_hash_corruption_or_changed_price_only_quarantines_affected_code(self): + good = entry() + for changed in ({**good['record'], 'version': 'bad'}, + {**good['record'], 'up_limit': '12.00'}, + {k: v for k, v in good['record'].items() if k != 'version'}): + self.value['entries'][A]['record'] = changed + self.assertEqual(set(cache.read_cache(DAY)), {B}) + + def test_bad_record_price_cannot_pass_by_preserving_previous_hash(self): + for changes in ({'pre_close': False}, {'up_limit': 0}, {'down_limit': 'NaN'}): + self.value['entries'][A]['record'] = {**record(), **changes} + self.assertEqual(set(cache.read_cache(DAY)), {B}) + + def test_source_status_and_fetched_timestamp_are_mandatory(self): + for changes in ({'source': 'computed'}, {'source': None}, {'status': 'UNSUPPORTED'}, + {'status': 'FAILED'}, {'fetched_at': ''}, {'fetched_at': 'garbage'}, + {'fetched_at': '2026-11-03T09:55:00'}, {'fetched_at': True}, + {'fetched_at': (NOW-timedelta(days=1)).isoformat()}, + {'fetched_at': (NOW+timedelta(seconds=1)).isoformat()}): + self.value['entries'][A] = entry(**changes) + self.assertEqual(set(cache.read_cache(DAY)), {B}) + + def test_timezone_conversion_checks_beijing_date(self): + self.value['entries'][A]['fetched_at'] = '2026-11-02T16:01:00+00:00' + self.assertIn(A, cache.read_cache(DAY)) + self.value['entries'][A]['fetched_at'] = '2026-11-02T15:59:00+00:00' + self.assertNotIn(A, cache.read_cache(DAY)) + + def test_code_key_mismatch_and_beijing_rows_are_filtered(self): + self.value['entries'][A]['record'] = record(B) + self.value['entries'][BJ] = entry() + self.assertEqual(set(cache.read_cache(DAY)), {B}) + self.assertEqual(cache.get_limits(DAY, [BJ]), {}) + self.assertEqual(cache.get_limits(DAY, ['920001.SH']), {}) + + def test_unsupported_requested_codes_do_not_read_any_source(self): + self.assertEqual(cache.get_limits(DAY, [BJ, '430001.BJ']), {}) + self.read.assert_not_called() + self.assert_no_legacy() + + def test_requested_filter_returns_only_selected_shanghai_shenzhen_codes(self): + self.assertEqual(set(cache.get_limits(DAY, [A, BJ])), {A}) + self.assertEqual(set(cache.get_limits(DAY)), {A, B}) + self.assertEqual(cache.get_limits(DAY, []), {}) + + def test_read_is_side_effect_free_and_does_not_extend_fetched_time(self): + original = copy.deepcopy(self.value) + first = cache.get_limits(DAY, [A]) + first[A]['up_limit'] = '99.00' + self.assertEqual(self.value, original) + self.assertEqual(cache.get_limits(DAY, [A])[A]['fetched_at'], original['entries'][A]['fetched_at']) + self.assertEqual(cache.get_limits(DAY, [A])[A]['up_limit'], '11.00') + + def test_every_read_observes_new_persisted_version_without_process_ttl(self): + old = cache.get_limits(DAY, [A])[A] + self.value['entries'][A]['record'] = record(high='11.01') + new = cache.get_limits(DAY, [A])[A] + self.assertNotEqual(old['version'], new['version']) + self.assertEqual(self.read.call_count, 2) + + def test_cache_read_failure_never_falls_back_to_shared_table(self): + self.read.side_effect = RuntimeError('local store unavailable') + self.assertEqual(cache.get_limits(DAY, [A]), {}) + self.assert_no_legacy() + + def test_feature_disabled_uses_old_source_and_filters_beijing(self): + self.enabled = False + self.legacy.return_value = [{'ts_code': A, 'up_limit': 11, 'down_limit': 9}, + {'ts_code': BJ, 'up_limit': 11, 'down_limit': 9}] + result = cache.get_limits(DAY, [A, BJ]) + self.assertEqual(set(result), {A}) + sql, params = self.legacy.call_args.args + self.assertIn('gp_stk_limit', sql) + self.assertIn('ts_code IN', sql) + self.assertNotIn(BJ, params.values()) + self.read.assert_not_called() + + def test_market_consumer_uses_real_cache_and_preserves_metadata(self): + quote = market.execution_quote(A, now=NOW) + self.assertEqual((quote['limit_up'], quote['limit_down']), ('11.00', '9.00')) + self.assertEqual(quote['limit_source'], 'tushare.stk_limit') + self.assertEqual(quote['limit_version'], record()['version']) + self.assertEqual(quote['limit_fetched_at'], entry()['fetched_at']) + self.assertEqual(quote['source'], 'tushare:rt_min:1MIN') + self.assert_no_legacy() + + def test_price_order_keeps_exact_limit_source_version_and_fetch_time(self): + quote = market.execution_quote(A, now=NOW) + result = policy.price_order(side='buy', quote=quote, now=NOW) + self.assertEqual(result['action'], 'FIRE') + for key in ('limit_source', 'limit_version', 'limit_fetched_at'): + self.assertEqual(result[key], quote[key]) + + def test_market_missing_or_failed_cache_waits_without_old_source(self): + for fail in (False, True): + self.value = None + self.read.side_effect = RuntimeError('unavailable') if fail else lambda key: (None, 0) + quote = market.execution_quote(A, now=NOW) + self.assertIsNone(quote['limit_up']) + self.assertIsNone(quote['limit_down']) + self.assertIn(quote.get('limit_source'), (None, 'unknown', 'unavailable')) + self.assertEqual(policy.price_order(side='buy', quote=quote, now=NOW)['action'], 'WAIT') + self.assert_no_legacy() + + def test_market_feature_off_uses_original_sql_without_cache(self): + self.enabled = False + self.market_legacy.return_value = [{'up_limit': 11, 'down_limit': 9}] + quote = market.execution_quote(A, now=NOW) + self.assertEqual(quote['limit_up'], 11) + self.assertEqual(quote['limit_source'], 'gp_stk_limit') + self.assertIsNone(quote['limit_version']) + self.market_legacy.assert_called_once() + self.read.assert_not_called() + + def test_market_rejects_beijing_before_loading_quotes_or_limits(self): + self.assertEqual(market.execution_quote(BJ, now=NOW), {}) + self.bars.assert_not_called() + self.read.assert_not_called() + self.assert_no_legacy() + + def test_rotation_consumer_uses_actual_cache_and_reuses_validated_limits(self): + limits = rotation_service._limits(DAY) + self.assertEqual(set(limits), {A, B}) + quote = market.execution_quote(A, now=NOW, limits=limits[A]) + self.assertEqual(self.read.call_count, 1) + self.assertEqual(quote['limit_source'], 'tushare.stk_limit') + self.assertEqual(quote['limit_version'], limits[A]['version']) + self.assertEqual(policy.price_order(side='sell', quote=quote, now=NOW)['action'], 'FIRE') + self.assert_no_legacy() + + def test_rotation_missing_and_failed_cache_never_falls_back(self): + self.value = None + self.assertEqual(rotation_service._limits(DAY), {}) + self.read.side_effect = RuntimeError('unavailable') + self.assertEqual(rotation_service._limits(DAY), {}) + self.assert_no_legacy() + + def test_rotation_feature_off_retains_old_source_with_exchange_sql_filter(self): + self.enabled = False + self.legacy.return_value = [{'ts_code': A, 'up_limit': 11, 'down_limit': 9}] + self.assertEqual(set(rotation_service._limits(DAY)), {A}) + sql, params = self.legacy.call_args.args + self.assertIn('gp_stk_limit', sql) + self.assertIn("ts_code LIKE '60____.SH'", sql) + self.assertIn("ts_code LIKE '30____.SZ'", sql) + self.assertEqual(params['day'], DAY) + self.read.assert_not_called() + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/scripts/test_price_limit_collector.py b/scripts/test_price_limit_collector.py new file mode 100644 index 0000000..6cc7c46 --- /dev/null +++ b/scripts/test_price_limit_collector.py @@ -0,0 +1,295 @@ +"""独立限制价采集的离线业务边界。HTTP、数据库、日历和时钟均使用内存桩。""" +import unit_test_guard # noqa: F401 +import contextlib +import copy +import unittest +from datetime import datetime, timedelta +from unittest.mock import patch +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from app.core import price_limits as core +from app.core.execution_policy import BEIJING +from app.services import price_limit_collector as worker, param_store +from config.settings import settings + +NOW = datetime(2026, 11, 3, 9, 0, tzinfo=BEIJING) +DAY = '2026-11-03' +A, B = '600000.SH', '000001.SZ' + + +def row(code=A, day=DAY, high='11.00'): + return {'ts_code': code, 'trade_date': day, 'pre_close': '10.00', + 'up_limit': high, 'down_limit': '9.00'} + + +class CollectorTests(unittest.TestCase): + def setUp(self): + self.stack = contextlib.ExitStack() + self.addCleanup(self.stack.close) + self.docs, self.params, self.rows, self.calls = {}, {}, [], [] + self.seconds = 0. + self.target = {'codes': [A, B], 'errors': [], 'sources': {A: ['持仓'], B: ['微盘候选']}} + self.result = lambda code: core.normalize_rows([row(code)], DAY, [code]) + self.stack.enter_context(patch.object(settings, 'PMS_TABLE_PREFIX', '')) + self.stack.enter_context(patch.object(settings, 'PMS_TUSHARE_TOKEN', 'test-secret-never-print')) + for name in ('get_bool', 'get_int', 'get_float'): + self.stack.enter_context(patch.object(param_store, name, side_effect=lambda key, default=None: + self.params.get(key, True if key == 'PMS_LIMIT_COLLECT_ENABLED' else default))) + self.stack.enter_context(patch.object(worker.tradedays, 'calendar_degraded', return_value=False)) + self.stack.enter_context(patch.object(worker.tradedays, 'is_trade_day', return_value=True)) + self.stack.enter_context(patch.object(worker.time, 'monotonic', side_effect=lambda: self.seconds)) + self.stack.enter_context(patch.object(worker.time, 'sleep', side_effect=self.advance)) + original_get = param_store.get + self.stack.enter_context(patch.object(param_store, 'get', side_effect=lambda key, default=None: + original_get(key, default) if key in param_store.SECRET_KEYS else self.params.get(key, default))) + self.collect = self.stack.enter_context(patch.object(worker.price_limit_targets, 'collect_targets', + side_effect=lambda day: copy.deepcopy(self.target))) + self.shared = self.stack.enter_context(patch.object(worker.db, 'fetch_all', side_effect=lambda *a, **kw: copy.deepcopy(self.rows))) + self.stack.enter_context(patch.object(worker.store, 'get', side_effect=lambda key: (copy.deepcopy(self.docs.get(key)), 1))) + self.stack.enter_context(patch.object(worker.store, 'locked', self.locked)) + self.fetch = self.stack.enter_context(patch.object(worker.price_limit_provider, 'fetch_limit', side_effect=self.provider)) + import requests + self.forbidden = [self.stack.enter_context(patch.object(obj, name, side_effect=AssertionError('真实外部访问'))) + for obj, name in [(requests, 'Session'), (worker.db, 'execute'), (worker.db, 'fetch_one')]] + + def tearDown(self): + for method in self.forbidden: + method.assert_not_called() + + @contextlib.contextmanager + def locked(self, key, kind, initial): + value = copy.deepcopy(self.docs.get(key, initial)) + yield value + self.docs[key] = value + + def advance(self, seconds): + self.seconds += seconds + + def provider(self, code, day, token, timeout_sec): + self.calls.append((code, day, self.seconds)) + self.assertEqual(day, DAY) + self.assertGreater(timeout_sec, 0) + return self.result(code) + + def entry(self, code=A): + return self.docs[worker.day_key(DAY)]['entries'][code] + + def run_round(self, now=NOW): + return worker.run_once(now) + + def test_missing_targets_fetched_and_persisted_with_source(self): + result = self.run_round() + self.assertEqual((result['status'], result['valid_count'], result['requests']), ('COMPLETE', 2, 2)) + self.assertEqual(self.entry()['source'], 'tushare.stk_limit') + self.assertEqual(self.entry()['record']['trade_date'], DAY) + self.assertEqual(self.docs[worker.day_key(DAY)]['target_sources'][A], ['持仓']) + self.assertNotIn('owner', self.docs[worker.LEASE_KEY]) + + def test_single_code_requests_are_globally_spaced(self): + self.run_round() + self.assertEqual(len(self.calls), 2) + self.assertGreaterEqual(self.calls[1][2]-self.calls[0][2], 1.2) + for code, _, _ in self.calls: + self.assertNotIn(',', code) + + def test_shared_current_values_avoid_redundant_http(self): + self.rows = [row(A), row(B)] + self.assertEqual(self.run_round()['valid_count'], 2) + self.fetch.assert_not_called() + self.assertEqual(self.entry()['source'], 'gp_stk_limit') + sql, bindings = self.shared.call_args.args + self.assertIn('ts_code IN', sql) + self.assertEqual(set(bindings.values()), {DAY, A, B}) + + def test_target_failure_keeps_other_targets(self): + self.target['errors'] = ['计划读取失败'] + result = self.run_round() + self.assertEqual((result['status'], result['valid_count']), ('PARTIAL', 2)) + self.assertEqual(result['errors'], ['计划读取失败']) + + def test_bj_excluded_before_shared_and_http(self): + self.target['codes'].append('920001.BJ') + result = self.run_round() + self.assertEqual(result['target_count'], 2) + self.assertNotIn('920001.BJ', self.shared.call_args.args[1].values()) + self.assertNotIn('920001.BJ', [c[0] for c in self.calls]) + + def test_wrong_day_shared_value_does_not_stop_provider(self): + self.rows = [row(A, '2026-11-02')] + self.run_round() + self.assertEqual(self.entry()['status'], 'VALID') + self.assertEqual(self.entry()['source'], 'tushare.stk_limit') + + def test_one_empty_response_keeps_other_valid_and_retries_later(self): + self.result = lambda c: core.normalize_rows([] if c == A else [row(c)], DAY, [c]) + r = self.run_round() + self.assertEqual((r['valid_count'], r['missing_count']), (1, 1)) + self.assertEqual(self.entry()['last_error'], 'NOT_PUBLISHED') + self.calls.clear() + self.run_round(NOW+timedelta(seconds=20)) + self.assertFalse(self.calls) + self.run_round(NOW+timedelta(seconds=70)) + self.assertEqual([c[0] for c in self.calls], [A]) + self.assertEqual(self.entry()['failures'], 2) + + def test_failed_first_does_not_starve_next_round(self): + self.params['PMS_LIMIT_ROUND_MAX_CODES'] = 1 + self.result = lambda c: core.normalize_rows([], DAY, [c]) + self.run_round() + first = self.calls[-1][0] + self.run_round(NOW+timedelta(seconds=10)) + self.assertNotEqual(first, self.calls[-1][0]) + + def test_max_failures_persist_across_rounds(self): + self.params['PMS_LIMIT_MAX_FAILURES_PER_DAY'] = 1 + self.result = lambda c: {'records': {}, 'invalid': {}, 'errors': ['HTTP_TIMEOUT']} + self.run_round() + self.calls.clear() + self.run_round(NOW+timedelta(minutes=20)) + self.assertFalse(self.calls) + + def test_transport_error_keeps_valid_content_and_original_timestamp(self): + self.run_round() + old = copy.deepcopy(self.entry()) + self.result = lambda c: {'records': {}, 'invalid': {}, 'errors': ['HTTP_TIMEOUT']} + self.run_round(NOW+timedelta(minutes=20)) + self.assertEqual(self.entry()['record'], old['record']) + self.assertEqual(self.entry()['fetched_at'], old['fetched_at']) + self.assertEqual(self.entry()['status'], 'VALID') + self.assertEqual(self.entry()['last_error'], 'HTTP_TIMEOUT') + + def test_explicit_bad_value_quarantines_existing_record(self): + self.run_round() + self.result = lambda c: core.normalize_rows([row(c, high='0')], DAY, [c]) + self.run_round(NOW+timedelta(minutes=20)) + self.assertEqual(self.entry()['status'], 'INVALID') + self.assertEqual(self.entry()['last_error'], 'UNSUPPORTED') + + def test_revised_value_keeps_prior_version(self): + self.run_round() + old = self.entry()['record']['version'] + self.result = lambda c: core.normalize_rows([row(c, high='11.20')], DAY, [c]) + self.run_round(NOW+timedelta(minutes=20)) + self.assertNotEqual(self.entry()['record']['version'], old) + self.assertEqual(self.entry()['revisions'][0]['record']['version'], old) + + def test_later_shared_old_record_cannot_override_tushare(self): + self.run_round() + old = self.entry()['record'] + self.rows = [row(A, high='11.10')] + self.run_round(NOW+timedelta(seconds=30)) + self.assertEqual(self.entry()['record'], old) + + def test_shared_failure_uses_provider(self): + self.shared.side_effect = RuntimeError('never-print') + r = self.run_round() + self.assertEqual(r['valid_count'], 2) + self.assertEqual(r['errors'], ['共享限制价读取失败']) + + def test_missing_token_still_uses_shared_data(self): + self.stack.enter_context(patch.object(settings, 'PMS_TUSHARE_TOKEN', '')) + self.rows = [row(A)] + r = self.run_round() + self.assertEqual(r['valid_count'], 1) + self.assertIn('采集密钥未配置', r['errors']) + self.fetch.assert_not_called() + + def test_real_prefix_disabled_and_calendar_window_guards(self): + with patch.object(settings, 'PMS_TABLE_PREFIX', 'real_'): + self.assertEqual(self.run_round()['status'], 'UNAUTHORIZED_INSTANCE') + self.params['PMS_LIMIT_COLLECT_ENABLED'] = False + self.assertEqual(self.run_round()['status'], 'DISABLED') + self.params.clear() + with patch.object(worker.tradedays, 'calendar_degraded', return_value=True): + self.assertEqual(self.run_round()['status'], 'CALENDAR_WAIT') + with patch.object(worker.tradedays, 'is_trade_day', return_value=False): + self.assertEqual(self.run_round()['status'], 'CALENDAR_WAIT') + self.assertEqual(self.run_round(NOW.replace(hour=8))['status'], 'OUTSIDE_WINDOW') + self.params['PMS_LIMIT_COLLECT_START'] = 'bad' + self.assertEqual(self.run_round()['status'], 'INVALID_WINDOW') + self.collect.assert_not_called() + + def test_invalid_runtime_values_do_not_fetch(self): + self.params['PMS_LIMIT_MIN_REQUEST_INTERVAL_SEC'] = 0 + self.assertEqual(self.run_round()['status'], 'INVALID_CONFIG') + self.collect.assert_not_called() + + def test_live_lease_skips_all_sources(self): + self.docs[worker.LEASE_KEY] = {'owner': 'other', 'until': (NOW+timedelta(seconds=30)).isoformat()} + self.assertEqual(self.run_round()['status'], 'BUSY') + self.collect.assert_not_called() + + def test_expired_lease_is_recovered(self): + self.docs[worker.LEASE_KEY] = {'owner': 'old', 'until': (NOW-timedelta(seconds=1)).isoformat()} + self.assertEqual(self.run_round()['valid_count'], 2) + + def test_lost_lease_does_not_overwrite_new_owner(self): + def replaced(c): + self.docs[worker.LEASE_KEY] = {'owner': 'new', 'until': (NOW+timedelta(seconds=300)).isoformat()} + return core.normalize_rows([row(c)], DAY, [c]) + self.result = replaced + self.assertEqual(self.run_round()['status'], 'LEASE_LOST') + self.assertNotIn(worker.day_key(DAY), self.docs) + self.assertEqual(self.docs[worker.LEASE_KEY]['owner'], 'new') + + def test_round_timeout_discards_late_response(self): + def slow(c): + self.advance(31) + return core.normalize_rows([row(c)], DAY, [c]) + self.result = slow + r = self.run_round() + self.assertEqual(r['valid_count'], 0) + self.assertIn('本轮采集超时,迟到响应未采纳', r['errors']) + self.assertEqual(self.entry()['status'], 'MISSING') + + def test_old_day_document_is_not_reused(self): + self.docs[worker.day_key(DAY)] = {'trade_date': '2026-11-02', 'entries': {A: {'status': 'VALID'}}} + self.run_round() + self.assertEqual(self.entry()['record']['trade_date'], DAY) + + def test_shared_bad_row_cannot_reset_http_backoff(self): + self.rows = [row(A, high='0')] + self.result = lambda c: {'records': {}, 'invalid': {}, 'errors': ['HTTP_TIMEOUT']} + self.run_round() + self.calls.clear() + self.run_round(NOW+timedelta(seconds=20)) + self.assertFalse(self.calls) + self.assertEqual(self.entry()['failures'], 1) + + def test_slow_first_request_does_not_starve_next_stock(self): + def slow(c): + self.advance(31) + return core.normalize_rows([row(c)], DAY, [c]) + self.result = slow + self.run_round() + first = self.calls[-1][0] + self.run_round(NOW+timedelta(seconds=40)) + self.assertNotEqual(first, self.calls[-1][0]) + + def test_empty_targets_no_market_wide_query(self): + self.target = {'codes': [], 'errors': [], 'sources': {}} + self.assertEqual(self.run_round()['target_count'], 0) + self.shared.assert_not_called() + self.fetch.assert_not_called() + + def test_one_bad_schedule_record_does_not_block_other_targets(self): + self.docs[worker.day_key(DAY)] = {'trade_date': DAY, 'entries': { + A: {'status': 'INVALID', 'failures': 'bad', 'last_attempt': [], 'revisions': {}, 'last_error': {'bad': True}}}} + self.assertEqual(self.run_round()['valid_count'], 2) + self.assertEqual(self.entry()['status'], 'VALID') + + def test_bad_entries_container_is_recovered(self): + self.docs[worker.day_key(DAY)] = {'trade_date': DAY, 'entries': []} + self.assertEqual(self.run_round()['valid_count'], 2) + + def test_secret_not_editable_or_readable_through_paramstore(self): + self.assertIn('PMS_TUSHARE_TOKEN', param_store.SECRET_KEYS) + self.assertNotIn('PMS_TUSHARE_TOKEN', param_store._editable_keys()) + self.assertEqual(param_store.get('PMS_TUSHARE_TOKEN'), '') + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_price_limit_provider.py b/scripts/test_price_limit_provider.py new file mode 100644 index 0000000..bba1e6d --- /dev/null +++ b/scripts/test_price_limit_provider.py @@ -0,0 +1,234 @@ +"""限制价HTTP客户端的内存桩回归;不连接供应商、数据库或代理。""" +import unit_test_guard +import contextlib +import io +import json +import os +import sys +import unittest +from datetime import date +from unittest.mock import patch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from app.services import price_limit_provider as provider + +CODE = '688701.SH' +DAY = '2026-11-03' +TOKEN = 'unit-only-secret-canary-do-not-log' + + +def payload(items=None, **extra): + return {'code': 0, 'data': {'fields': provider.FIELDS.split(','), + 'items': [[CODE, '20261103', 10, 12, 8]] if items is None else items}, **extra} + + +class FakeResponse: + def __init__(self, body=None, *, status=200, headers=None, chunks=None, failure=None): + self.status_code = status + self.headers = headers or {} + self.chunks = chunks if chunks is not None else [json.dumps(payload() if body is None else body).encode()] + self.failure = failure + self.closed = False + self.reads = 0 + + def iter_content(self, chunk_size): + self.chunk_size = chunk_size + for chunk in self.chunks: + self.reads += 1 + yield chunk + if self.failure: + raise self.failure + + def close(self): + self.closed = True + + +class FakeSession: + def __init__(self, response, failure=None): + self.response, self.failure = response, failure + self.trust_env = True + self.closed = False + self.calls = [] + + def post(self, url, **kwargs): + self.calls.append((url, kwargs, self.trust_env)) + if self.failure: + raise self.failure + return self.response + + def close(self): + self.closed = True + + +class ProviderTests(unittest.TestCase): + def request(self, response=None, *, code=CODE, day=DAY, token=TOKEN, timeout_sec=5, + failure=None, clock=None): + response = response or FakeResponse() + session = FakeSession(response, failure) + with contextlib.ExitStack() as stack: + factory = stack.enter_context(patch.object(provider.requests, 'Session', return_value=session)) + if clock is not None: + stack.enter_context(patch.object(provider.time, 'monotonic', side_effect=clock)) + result = provider.fetch_limit(code, day, token, timeout_sec) + return result, response, session, factory + + def test_single_security_contract_and_normalized_date(self): + result, response, session, _ = self.request(code=' 688701.sh ', day=date(2026, 11, 3)) + self.assertEqual(result['errors'], []) + self.assertEqual(list(result['records']), [CODE]) + self.assertEqual(result['records'][CODE]['up_limit'], '12.00') + self.assertEqual(len(session.calls), 1) + url, kwargs, trust = session.calls[0] + self.assertEqual(url, 'https://api.tushare.pro') + self.assertEqual(kwargs['json']['api_name'], 'stk_limit') + self.assertEqual(kwargs['json']['params'], {'ts_code': CODE, 'trade_date': '20261103'}) + self.assertEqual(kwargs['json']['fields'], provider.FIELDS) + self.assertTrue(kwargs['stream']) + self.assertFalse(kwargs['allow_redirects']) + self.assertFalse(trust) + self.assertEqual(kwargs['timeout'].total, 5.) + self.assertEqual(kwargs['timeout'].connect_timeout, 2.) + self.assertTrue(response.closed and session.closed) + + def test_empty_success_remains_empty_without_error(self): + result, response, session, _ = self.request(FakeResponse(payload([]))) + self.assertEqual(result, {'records': {}, 'invalid': {}, 'errors': []}) + self.assertTrue(response.closed and session.closed) + + def test_code_rejections_never_create_session(self): + for code in ('', None, '688701.SH,600000.SH', '999999.BJ', '000001.SH', 'HK00700', '*'): + with self.subTest(code=code): + result, _, _, factory = self.request(code=code) + self.assertEqual(result['errors'], ['INVALID_REQUEST_CODE']) + factory.assert_not_called() + + def test_invalid_date_never_creates_session(self): + for day in (None, True, '2026-02-30', '2026113', '2026-11-03 extra', []): + with self.subTest(day=day): + result, _, _, factory = self.request(day=day) + self.assertEqual(result['errors'], ['INVALID_TARGET_DATE']) + factory.assert_not_called() + + def test_missing_token_never_creates_session(self): + for token in ('', ' \n ', None, False, 123): + result, _, _, factory = self.request(token=token) + self.assertEqual(result['errors'], ['MISSING_PROVIDER_TOKEN']) + factory.assert_not_called() + + def test_invalid_timeout_never_creates_session(self): + for value in (0, -1, float('nan'), float('inf'), True, 'bad', None): + result, _, _, factory = self.request(timeout_sec=value) + self.assertEqual(result['errors'], ['INVALID_HTTP_TIMEOUT']) + factory.assert_not_called() + + def test_short_timeout_caps_connect(self): + _, _, session, _ = self.request(timeout_sec=.25) + timeout = session.calls[0][1]['timeout'] + self.assertEqual(timeout.total, .25) + self.assertEqual(timeout.connect_timeout, .25) + + def test_redirect_and_http_failure_never_read_body(self): + for status in (301, 302, 307, 401, 429, 500): + result, response, session, _ = self.request(FakeResponse(status=status)) + self.assertEqual(result['errors'], ['HTTP_ERROR']) + self.assertEqual(response.reads, 0) + self.assertTrue(response.closed and session.closed) + self.assertEqual(len(session.calls), 1) + + def test_provider_error_never_returns_message_or_token(self): + stream = io.StringIO() + with contextlib.redirect_stdout(stream), contextlib.redirect_stderr(stream): + result, _, _, _ = self.request(FakeResponse(payload(code=-2001, msg=TOKEN))) + self.assertEqual(result['errors'], ['PROVIDER_ERROR']) + self.assertNotIn(TOKEN, json.dumps(result)+stream.getvalue()) + + def test_transport_errors_do_not_echo_details_or_retry(self): + for failure, code in ((provider.requests.exceptions.Timeout(TOKEN), 'HTTP_TIMEOUT'), + (provider.requests.exceptions.ConnectionError(TOKEN), 'HTTP_ERROR'), + (RuntimeError(TOKEN), 'HTTP_ERROR')): + stream = io.StringIO() + with contextlib.redirect_stdout(stream), contextlib.redirect_stderr(stream): + result, _, session, _ = self.request(failure=failure) + self.assertEqual(result['errors'], [code]) + self.assertEqual(len(session.calls), 1) + self.assertTrue(session.closed) + self.assertNotIn(TOKEN, json.dumps(result)+stream.getvalue()) + + def test_stream_timeout_closes_both_resources(self): + result, response, session, _ = self.request(FakeResponse(chunks=[b'{'], + failure=provider.requests.exceptions.ReadTimeout(TOKEN))) + self.assertEqual(result['errors'], ['HTTP_TIMEOUT']) + self.assertTrue(response.closed and session.closed) + + def test_requests_wrapped_stream_timeout_is_still_a_timeout(self): + from urllib3.exceptions import ReadTimeoutError + failure = provider.requests.exceptions.ConnectionError(ReadTimeoutError(None, TOKEN, TOKEN)) + result, response, session, _ = self.request(FakeResponse(chunks=[b'{'], failure=failure)) + self.assertEqual(result['errors'], ['HTTP_TIMEOUT']) + self.assertNotIn(TOKEN, json.dumps(result)) + self.assertTrue(response.closed and session.closed) + + def test_declared_size_rejects_before_reading(self): + result, response, session, _ = self.request(FakeResponse(headers={'Content-Length': '65537'})) + self.assertEqual(result['errors'], ['RESPONSE_TOO_LARGE']) + self.assertEqual(response.reads, 0) + self.assertTrue(response.closed and session.closed) + + def test_stream_limit_applies_without_or_despite_content_length(self): + for headers in ({}, {'Content-Length': '1'}): + result, response, session, _ = self.request(FakeResponse(headers=headers, + chunks=[b' ' * 65536, b'x', b'never-read'])) + self.assertEqual(result['errors'], ['RESPONSE_TOO_LARGE']) + self.assertEqual(response.reads, 2) + self.assertTrue(response.closed and session.closed) + + def test_exact_size_valid_body_is_accepted(self): + body = json.dumps(payload()).encode() + body += b' ' * (65536-len(body)) + result, _, _, _ = self.request(FakeResponse(chunks=[body], headers={'Content-Length': '65536'})) + self.assertEqual(list(result['records']), [CODE]) + + def test_deadline_after_headers_stops_before_read(self): + result, response, session, _ = self.request(clock=[0., 5.]) + self.assertEqual(result['errors'], ['HTTP_TIMEOUT']) + self.assertEqual(response.reads, 0) + self.assertTrue(response.closed and session.closed) + + def test_deadline_during_stream_stops_without_more_chunks(self): + result, response, _, _ = self.request(FakeResponse(chunks=[b'{', b'}', b'never-read']), + clock=[0., 1., 2., 5.]) + self.assertEqual(result['errors'], ['HTTP_TIMEOUT']) + self.assertEqual(response.reads, 2) + self.assertTrue(response.closed) + + def test_invalid_json_and_headers_are_safe(self): + for response in (FakeResponse(chunks=[TOKEN.encode()]), FakeResponse(chunks=[b'\xff']), + FakeResponse(headers={'Content-Length': TOKEN}), + FakeResponse(headers={'Content-Length': '-1'})): + result, response, session, _ = self.request(response) + self.assertEqual(result['errors'], ['INVALID_PROVIDER_RESPONSE']) + self.assertNotIn(TOKEN, json.dumps(result)) + self.assertTrue(response.closed and session.closed) + + def test_foreign_security_and_wrong_day_are_not_accepted(self): + result, _, _, _ = self.request(FakeResponse(payload([ + ['600000.SH', '20261103', 10, 11, 9], + [CODE, '20261102', 10, 12, 8], + ]))) + self.assertEqual(result['records'], {}) + self.assertEqual(result['invalid'], {'600000.SH': 'NOT_REQUESTED', CODE: 'DATE_MISMATCH'}) + + def test_decoder_receives_nonempty_single_code_allowlist(self): + with patch.object(provider.price_limits, 'provider_decode', wraps=provider.price_limits.provider_decode) as decode: + self.request() + self.assertEqual(decode.call_args.kwargs['requested_codes'], [CODE]) + + def test_environment_proxy_is_not_trusted(self): + with patch.dict(os.environ, {'HTTPS_PROXY': 'https://'+TOKEN, 'NETRC': TOKEN}): + result, _, session, _ = self.request() + self.assertEqual(result['errors'], []) + self.assertFalse(session.calls[0][2]) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/scripts/test_price_limit_targets.py b/scripts/test_price_limit_targets.py new file mode 100644 index 0000000..3beebb5 --- /dev/null +++ b/scripts/test_price_limit_targets.py @@ -0,0 +1,230 @@ +"""限制价目标采集:全部资料来自内存桩,禁止数据库、HTTP和模型调用。""" +import unit_test_guard # noqa: F401 +from contextlib import ExitStack +import copy +import datetime as dt +import hashlib +import json +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from app.core import micro_rotation as core, strategy_book as books +from app.services import price_limit_targets as targets + + +DAY = '2026-11-04' + + +def snapshot(*, rule=core.EXECUTION_RULE, day=DAY, start=600100, ready=True, frozen=True): + members = [{'code': f'{start+i}.SH', 'rank': i+1, 'eligible': True, 'close': 10., + 'total_mv_wan': i+1} for i in range(100)] + value = {'rule_version': rule, 'rule_parameters': core.rule_parameters(rule), + 'basket_size': 50, 'data_date': '2026-11-03', 'effective_date': day, + 'signal_ready': True, 'basket_ready': ready, + 'signal': {'date': '2026-11-03', 'ready': True, 'low': False, + 'high': False, 'growth_break': False}, + 'members': members[:50], 'source_hashes': {'stocks': 'fixed'}, 'errors': []} + if rule == core.EXECUTION_RULE: + value.update(candidates=members, candidate_count=len(members)) + value['snapshot_version'] = hashlib.sha256(json.dumps(value, sort_keys=True, + ensure_ascii=False, separators=(',', ':')).encode()).hexdigest() + value['frozen'] = frozen + return value + + +class TargetCollectionTests(unittest.TestCase): + def setUp(self): + self.stack = ExitStack() + self.addCleanup(self.stack.close) + self.rows = {'positions': [], 'instructions': [], 'orders': [], 'plans': [], 'proposals': []} + self.state = None + self.saved = [] + self.config = {'rule_version': core.EXECUTION_RULE, 'candidate_pool_size': 100, + 'url': 'http://memory.invalid'} + self.monitored = snapshot() + self.readers = {} + for obj, method, name in ( + (targets.pms_repo, 'list_positions', 'positions'), + (targets.pms_repo, 'list_instructions', 'instructions'), + (targets.qmt_repo, 'list_orders', 'orders'), + (targets.pms_repo, 'list_plans', 'plans'), + (targets.pms_repo, 'list_proposals', 'proposals')): + def read(*args, source=name, **kwargs): + self.assertEqual(books.CURRENT.get(), 'all') + return copy.deepcopy(self.rows[source]) + self.readers[name] = self.stack.enter_context(patch.object(obj, method, side_effect=read)) + self.get_state = self.stack.enter_context(patch.object(targets.rotation_repo, 'get', + side_effect=lambda key: (copy.deepcopy(self.state), 1))) + self.recent = self.stack.enter_context(patch.object(targets.rotation_repo, 'recent', + side_effect=lambda *args, **kwargs: copy.deepcopy(self.saved))) + self.configuration = self.stack.enter_context(patch.object(targets.rotation_service, 'configuration', + side_effect=lambda: copy.deepcopy(self.config))) + self.pull = self.stack.enter_context(patch.object(targets.rotation_service, 'pull', + side_effect=lambda day: copy.deepcopy(self.monitored))) + # 即使调用了会被业务层捕获的外部入口,也在 tearDown 显式失败。 + from app.db import session as db + import requests + self.forbidden = [self.stack.enter_context(patch.object(obj, name, + side_effect=AssertionError('测试禁止真实外部调用'))) + for obj, name in ((db, 'fetch_all'), (db, 'fetch_one'), (db, 'execute'), + (requests, 'get'), (requests, 'post'))] + + def tearDown(self): + for mock in self.forbidden: + mock.assert_not_called() + + def test_all_sources_and_both_books_merge_with_provenance(self): + self.rows['positions'] = [{'ts_code': '600000.SH', 'book_id': 'original'}, + {'ts_code': '000001.SZ', 'book_id': 'micro30'}] + self.rows['instructions'] = [{'ts_code': '300001.SZ', 'status': 'WAIT_REVIEW'}] + self.rows['orders'] = [{'ts_code': '688001.SH', 'parent_id': 'terminal-parent'}] + self.rows['plans'] = [{'ts_code': '600000.SH'}, {'ts_code': '600002.SH'}] + self.rows['proposals'] = [{'ts_code': '600003.SH'}] + self.state = {'rotation': {'target': ['600004.SH']}} + self.saved = [{'value': snapshot(start=601100)}] + out = targets.collect_targets(DAY) + self.assertEqual(out['errors'], []) + self.assertTrue({'600000.SH', '000001.SZ', '300001.SZ', '688001.SH', + '600002.SH', '600003.SH', '600004.SH', '600199.SH', '601199.SH'} <= set(out['codes'])) + self.assertEqual(out['sources']['600000.SH'], ['执行计划', '策略持仓']) + self.assertEqual(out['codes'], sorted(set(out['codes']))) + self.assertEqual(books.CURRENT.get(), books.ORIGINAL) + + def test_status_contract_covers_reviews_plans_and_accepted_proposals(self): + targets.collect_targets(DAY) + self.assertIn('WAIT_REVIEW', self.readers['instructions'].call_args.kwargs['statuses']) + self.assertEqual(self.readers['plans'].call_args.kwargs['statuses'], ('PENDING', 'GATED', 'EXECUTING')) + self.assertEqual(self.readers['proposals'].call_args.kwargs['statuses'], ('WAIT_USER', 'ACCEPTED')) + self.assertEqual(self.readers['orders'].call_args.kwargs['statuses'], targets.qmt_repo.LIVE) + for key in ('instructions', 'plans', 'orders', 'proposals'): + self.assertEqual(self.readers[key].call_args.kwargs['limit'], 5000) + + def test_normalizes_and_excludes_out_of_scope_at_entry(self): + self.rows['positions'] = [{'ts_code': code} for code in + (' 600000.sh ', '000001.sz', '920001.BJ', '830001.BJ', '000001.SH', '', None)] + self.state = {'rotation': {'target': ['920001.BJ', '688001.SH']}} + out = targets.collect_targets(DAY) + self.assertTrue({'600000.SH', '000001.SZ', '688001.SH'} <= set(out['codes'])) + self.assertFalse(any(code.endswith('.BJ') for code in out['codes'])) + self.assertNotIn('000001.SH', out['codes']) + + def test_each_failed_source_keeps_other_known_targets_and_hides_exception_text(self): + self.rows['positions'] = [{'ts_code': '600000.SH'}] + self.readers['instructions'].side_effect = RuntimeError('private connection text') + self.readers['plans'].side_effect = RuntimeError('private connection text') + self.pull.side_effect = RuntimeError('private connection text') + out = targets.collect_targets(DAY) + self.assertEqual(out['codes'], ['600000.SH']) + self.assertEqual(out['errors'], ['待执行指令读取失败', '执行计划读取失败', '微盘监控快照读取失败']) + self.assertNotIn('private', str(out)) + + def test_cap_is_reported_at_exact_limit_without_discarding_known_codes(self): + self.rows['instructions'] = [{'ts_code': '600000.SH'}]*5000 + out = targets.collect_targets(DAY) + self.assertIn('待执行指令达到读取上限,结果可能不完整', out['errors']) + self.assertIn('600000.SH', out['codes']) + + def test_snapshot_repo_real_cap_is_reported(self): + self.saved = [{'value': snapshot(day='2026-11-05')}]*1000 + out = targets.collect_targets(DAY) + self.recent.assert_called_once_with('snapshot', limit=1000) + self.assertIn('近期微盘快照达到读取上限,结果可能不完整', out['errors']) + + def test_wrong_day_and_old_rule_saved_snapshots_are_ignored(self): + self.saved = [{'value': snapshot(day='2026-11-05', start=601100)}, + {'value': snapshot(rule=core.RULE, start=602100)}] + out = targets.collect_targets(DAY) + self.assertEqual(len(out['codes']), 100) + self.assertEqual(out['errors'], []) + + def test_tampered_saved_candidate_pool_is_not_used(self): + value = snapshot(start=601100) + value['candidates'][90]['code'] = '602999.SH' + self.saved = [{'value': value}] + out = targets.collect_targets(DAY) + self.assertNotIn('602999.SH', out['codes']) + self.assertIn('微盘保存快照校验未通过', out['errors']) + + def test_valid_unfrozen_monitor_pool_can_prewarm_without_authorizing_trade(self): + self.monitored = snapshot(frozen=False) + out = targets.collect_targets(DAY) + self.pull.assert_called_once_with(DAY) + self.assertEqual(len(out['codes']), 100) + self.assertEqual(out['sources']['600199.SH'], ['微盘监控快照']) + self.assertEqual(out['errors'], []) + + def test_monitor_wrong_day_or_rule_is_reported(self): + for value in (snapshot(day='2026-11-05'), snapshot(rule=core.RULE), None): + with self.subTest(value=value): + self.monitored = value + out = targets.collect_targets(DAY) + self.assertEqual(out['codes'], []) + self.assertIn('微盘监控快照日期或规则不符', out['errors']) + + def test_unready_monitor_does_not_treat_unchecked_candidates_as_eligible(self): + self.monitored = snapshot(ready=False) + out = targets.collect_targets(DAY) + self.assertEqual(out['codes'], []) + self.assertIn('微盘监控快照名单尚未就绪', out['errors']) + + def test_v2_only_collects_primary_members(self): + self.config['rule_version'] = core.RULE + self.monitored = snapshot(rule=core.RULE) + # 旧契约未签入摘要的附加字段不能被当作后备候选。 + self.monitored['candidates'] = [{'code': '602999.SH'}] + self.assertEqual(len(targets.collect_targets(DAY)['codes']), 50) + + def test_configuration_failure_preserves_holdings_and_state(self): + self.rows['positions'] = [{'ts_code': '600000.SH'}] + self.state = {'rotation': {'target': ['600001.SH']}} + self.configuration.side_effect = RuntimeError('offline') + out = targets.collect_targets(DAY) + self.assertEqual(out['codes'], ['600000.SH', '600001.SH']) + self.assertIn('微盘运行规则读取失败', out['errors']) + self.pull.assert_not_called() + + def test_missing_monitor_url_preserves_saved_snapshot(self): + self.config['url'] = '' + self.saved = [{'value': snapshot(start=601100)}] + out = targets.collect_targets(DAY) + self.assertIn('601199.SH', out['codes']) + self.assertIn('微盘监控地址未配置', out['errors']) + self.pull.assert_not_called() + + def test_malformed_source_rows_do_not_drop_valid_rows(self): + self.rows['positions'] = [None, {'ts_code': '600000.SH'}] + self.rows['plans'] = {'ts_code': '600001.SH'} + self.state = {'rotation': {'target': '600002.SH'}} + out = targets.collect_targets(DAY) + self.assertIn('600000.SH', out['codes']) + self.assertIn('策略持仓记录格式异常', out['errors']) + self.assertIn('执行计划记录格式异常', out['errors']) + self.assertIn('微盘执行目标读取失败', out['errors']) + + def test_next_collection_discovers_intraday_new_plan(self): + before = targets.collect_targets(DAY) + self.rows['plans'].append({'ts_code': '600009.SH'}) + after = targets.collect_targets(DAY) + self.assertNotIn('600009.SH', before['codes']) + self.assertIn('600009.SH', after['codes']) + self.assertEqual(after['sources']['600009.SH'], ['执行计划']) + + def test_invalid_day_makes_no_source_call(self): + for day in ('20261104', '2026-02-30', None, '2026-11-04T09:00:00'): + self.assertEqual(targets.collect_targets(day), + {'codes': [], 'errors': ['目标交易日格式无效'], 'sources': {}}) + for mock in self.readers.values(): + mock.assert_not_called() + self.configuration.assert_not_called() + self.pull.assert_not_called() + + def test_date_object_uses_iso_day(self): + self.assertEqual(len(targets.collect_targets(dt.date(2026, 11, 4))['codes']), 100) + self.pull.assert_called_once_with(DAY) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_price_limits_core.py b/scripts/test_price_limits_core.py new file mode 100644 index 0000000..802f6ea --- /dev/null +++ b/scripts/test_price_limits_core.py @@ -0,0 +1,210 @@ +"""限制价主源纯逻辑回归。虚构数据、无配置、无网络。""" +import unit_test_guard +import copy +from datetime import date, datetime +from decimal import Decimal, localcontext +import json +from pathlib import Path +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from app.core.price_limits import MAX_ROWS, REQUIRED_FIELDS, normalize_rows, provider_decode, supported + +DAY = '2026-11-03' +CODE = '600000.SH' +OTHER = '000001.SZ' + + +def row(**changes): + return {'trade_date': '20261103', 'ts_code': CODE, 'pre_close': '10.00', + 'up_limit': '11.00', 'down_limit': '9.00', **changes} + + +def body(rows=None, fields=None): + fields = list(fields or REQUIRED_FIELDS) + rows = [row()] if rows is None else rows + return {'code': 0, 'msg': 'must-not-be-echoed', + 'data': {'fields': fields, 'items': [[value.get(field) for field in fields] for value in rows]}} + + +class PriceLimitCoreTests(unittest.TestCase): + def test_supported_codes_are_only_shenzhen_shanghai_a_shares(self): + for code in ('600000.SH', '688001.SH', '000001.SZ', '300001.SZ', ' 600000.sh '): + self.assertTrue(supported(code)) + for code in ('920001.BJ', '430001.BJ', '920001.SH', '600000.SZ', '000001.SH', + 'SH600000', '600000', '510300.SH', '888888.SH', None, True): + self.assertFalse(supported(code)) + + def test_dates_normalize_without_substituting_today(self): + for target in ('20261103', 20261103, DAY, date(2026, 11, 3), datetime(2026, 11, 3)): + for source in ('20261103', 20261103, DAY, date(2026, 11, 3)): + result = normalize_rows([row(trade_date=source)], target) + self.assertEqual(result['records'][CODE]['trade_date'], DAY) + + def test_invalid_target_date_rejects_entire_batch(self): + for value in (None, True, 20261103.0, '2026-02-30', '202611', '2026-11-03T00:00:00'): + result = normalize_rows([row()], value) + self.assertEqual(result['errors'], ['INVALID_TARGET_DATE']) + self.assertEqual(result['records'], {}) + + def test_wrong_day_only_quarantines_affected_security(self): + result = normalize_rows([row(trade_date='20261102'), row(ts_code=OTHER)], DAY) + self.assertEqual(result['invalid'], {CODE: 'DATE_MISMATCH'}) + self.assertEqual(set(result['records']), {OTHER}) + + def test_correct_date_cannot_hide_same_security_wrong_date(self): + for values in ([row(), row(trade_date='20261102')], [row(trade_date='20261102'), row()]): + result = normalize_rows(values, DAY) + self.assertEqual(result['records'], {}) + self.assertEqual(result['invalid'][CODE], 'DATE_MISMATCH') + + def test_malformed_row_date_is_not_accepted(self): + result = normalize_rows([row(trade_date='20261340')], DAY) + self.assertEqual(result['invalid'], {CODE: 'INVALID_DATE'}) + + def test_nan_bool_infinity_and_negative_prices_rejected(self): + for field in ('pre_close', 'up_limit', 'down_limit'): + for bad in (float('nan'), float('inf'), Decimal('NaN'), Decimal('sNaN'), + Decimal('Infinity'), True, False, -1, None, 'oops', '1e999999'): + with self.subTest(field=field, bad=repr(bad)): + result = normalize_rows([row(**{field: bad})], DAY) + self.assertEqual(result['records'], {}) + self.assertEqual(result['invalid'], {CODE: 'INVALID_PRICE'}) + + def test_zero_limits_are_unsupported_without_fabrication(self): + for changes in ({'up_limit': 0}, {'down_limit': '0'}, {'up_limit': 0, 'down_limit': 0}): + result = normalize_rows([row(**changes)], DAY) + self.assertEqual(result['invalid'], {CODE: 'UNSUPPORTED'}) + self.assertFalse(result['records']) + + def test_preclose_must_be_positive_and_within_inclusive_limits(self): + for close, error in ((0, 'INVALID_PRE_CLOSE'), ('8.99', 'PRE_CLOSE_OUTSIDE_LIMITS'), + ('11.01', 'PRE_CLOSE_OUTSIDE_LIMITS')): + self.assertEqual(normalize_rows([row(pre_close=close)], DAY)['invalid'][CODE], error) + for close in ('9.00', '11.00'): + self.assertIn(CODE, normalize_rows([row(pre_close=close)], DAY)['records']) + + def test_equal_or_inverted_limits_are_rejected(self): + for changes in ({'down_limit': 11}, {'down_limit': 12}): + self.assertEqual(normalize_rows([row(**changes)], DAY)['invalid'][CODE], 'INVALID_RANGE') + + def test_subcent_values_are_rejected_instead_of_rounded(self): + for field in ('pre_close', 'up_limit', 'down_limit'): + result = normalize_rows([row(**{field: '10.001'})], DAY) + self.assertEqual(result['invalid'][CODE], 'INVALID_PRECISION') + result = normalize_rows([row(pre_close=10.000000000000002)], DAY) + self.assertEqual(result['invalid'][CODE], 'INVALID_PRECISION') + + def test_decimal_equivalents_have_same_content_version(self): + expected = normalize_rows([row()], DAY)['records'][CODE] + equivalents = row(pre_close=10, up_limit=Decimal('11.0000'), down_limit=9.0) + actual = normalize_rows([equivalents], DAY)['records'][CODE] + self.assertEqual(expected, actual) + self.assertEqual(expected['pre_close'], '10.00') + self.assertEqual(len(expected['version']), 64) + json.dumps(actual, allow_nan=False) + changed = normalize_rows([row(up_limit='11.01')], DAY)['records'][CODE] + self.assertNotEqual(expected['version'], changed['version']) + + def test_decimal_context_does_not_change_price_validation(self): + with localcontext() as context: + context.prec = 2 + self.assertIn(CODE, normalize_rows([row(pre_close='10.01')], DAY)['records']) + + def test_identical_duplicates_deduplicate_but_conflicts_poison_security(self): + result = normalize_rows([row(), row(pre_close=10.0)], DAY) + self.assertEqual(len(result['records']), 1) + self.assertEqual(result['invalid'], {}) + for values in ([row(), row(up_limit='12.00'), row()], + [row(up_limit='12.00'), row(), row(up_limit='12.00')]): + result = normalize_rows(values+[row(ts_code=OTHER)], DAY) + self.assertEqual(result['invalid'], {CODE: 'DUPLICATE_CONFLICT'}) + self.assertEqual(set(result['records']), {OTHER}) + + def test_bad_price_row_cannot_be_hidden_by_good_duplicate(self): + for values in ([row(), row(pre_close=False)], [row(pre_close=False), row()]): + result = normalize_rows(values, DAY) + self.assertFalse(result['records']) + self.assertEqual(result['invalid'][CODE], 'INVALID_PRICE') + + def test_requested_scope_rejects_unknown_return_code(self): + result = normalize_rows([row(), row(ts_code=OTHER)], DAY, [CODE]) + self.assertEqual(result['invalid'], {OTHER: 'NOT_REQUESTED'}) + self.assertEqual(set(result['records']), {CODE}) + for codes in (None, []): + self.assertEqual(len(normalize_rows([row(), row(ts_code=OTHER)], DAY, codes)['records']), 2) + + def test_beijing_is_rejected_at_entry_without_examining_its_prices(self): + result = normalize_rows([row(ts_code='920001.BJ', pre_close=False), row()], DAY) + self.assertEqual(result['invalid'], {'920001.BJ': 'OUT_OF_SCOPE'}) + self.assertEqual(set(result['records']), {CODE}) + + def test_missing_local_field_is_isolated_and_input_is_unchanged(self): + missing = row() + missing.pop('pre_close') + values = [missing, row(ts_code=OTHER)] + snapshot = copy.deepcopy(values) + result = normalize_rows(values, DAY) + self.assertEqual(result['invalid'], {CODE: 'MISSING_FIELD'}) + self.assertEqual(set(result['records']), {OTHER}) + self.assertEqual(values, snapshot) + + def test_invalid_collection_structure_is_top_level_error(self): + for rows in (None, {}, 'secret', [row(), None]): + result = normalize_rows(rows, DAY) + self.assertTrue(result['errors']) + self.assertFalse(result['records']) + for requested in ('600000.SH', ['920001.BJ'], [False]): + self.assertEqual(normalize_rows([row()], DAY, requested)['errors'], ['INVALID_REQUESTED_CODES']) + + def test_provider_field_order_and_extra_fields_are_supported(self): + fields = ['unused']+list(reversed(REQUIRED_FIELDS)) + result = provider_decode(body(fields=fields), DAY, [CODE]) + self.assertEqual(result, normalize_rows([row()], DAY, [CODE])) + + def test_provider_errors_never_echo_secret_message_or_arbitrary_code(self): + secret = 'secret-token-must-not-appear' + for value in ({'code': -2001, 'msg': secret}, {'code': False, 'data': secret}, + {'code': '0', 'msg': secret}, {'code': 0, 'data': secret}, None): + result = provider_decode(value, DAY, [CODE]) + self.assertTrue(result['errors']) + self.assertNotIn(secret, json.dumps(result)) + result = normalize_rows([row(ts_code=secret)], DAY) + self.assertEqual(result['invalid'], {'row:1': 'INVALID_CODE'}) + self.assertNotIn(secret, json.dumps(result)) + + def test_provider_missing_duplicate_or_invalid_fields_reject_batch(self): + for fields in ([], ['ts_code'], list(REQUIRED_FIELDS)+['ts_code'], + list(REQUIRED_FIELDS)+[False], list(REQUIRED_FIELDS)+[[]]): + value = body() + value['data']['fields'] = fields + result = provider_decode(value, DAY) + self.assertTrue(result['errors']) + self.assertFalse(result['records']) + + def test_provider_item_structure_rejects_entire_batch(self): + for item in (None, {}, 'not-a-row', [CODE], list(range(len(REQUIRED_FIELDS)+1))): + value = body() + value['data']['items'].append(item) + result = provider_decode(value, DAY) + self.assertEqual(result['errors'], ['INVALID_PROVIDER_ITEM_STRUCTURE']) + self.assertFalse(result['records']) + + def test_provider_maximum_rows_and_empty_response(self): + value = body() + value['data']['items'] *= MAX_ROWS + self.assertIn(CODE, provider_decode(value, DAY)['records']) + value['data']['items'].append(value['data']['items'][0]) + self.assertEqual(provider_decode(value, DAY)['errors'], ['TOO_MANY_ROWS']) + self.assertEqual(normalize_rows([row()]*(MAX_ROWS+1), DAY)['errors'], ['TOO_MANY_ROWS']) + self.assertEqual(provider_decode(body(rows=[]), DAY), {'records': {}, 'invalid': {}, 'errors': []}) + + def test_provider_passes_through_per_security_quarantine(self): + result = provider_decode(body([row(trade_date='20261102'), row(ts_code=OTHER)]), DAY, [CODE, OTHER]) + self.assertEqual(result['invalid'], {CODE: 'DATE_MISMATCH'}) + self.assertEqual(set(result['records']), {OTHER}) + + +if __name__ == '__main__': + unittest.main(verbosity=2)