109 lines
4.2 KiB
Python
109 lines
4.2 KiB
Python
"""沪深 A 股样本边界;只规范化代码,不推断上市资格或补造资料。"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
|
||
# 与项目证券入口一致:沪市 60/68、深市 00/30;显式市场必须与号段相符。
|
||
_CODE = re.compile(r'^(?:(SH|SZ)([0-9]{6})|([0-9]{6})[.](SH|SZ)|([0-9]{6}))$')
|
||
_SQL_CODE = ('^(SH(60|68)[0-9]{4}|(60|68)[0-9]{4}[.]SH|(60|68)[0-9]{4}'
|
||
'|SZ(00|30)[0-9]{4}|(00|30)[0-9]{4}[.]SZ|(00|30)[0-9]{4})$')
|
||
_CODE_FIELDS = ('ts_code', 'stock_code', 'symbol', 'code')
|
||
_MEMBER_FIELDS = frozenset(('members', 'quiet', 'moved_members', 'constituents', 'stocks',
|
||
'ts_codes', 'stock_codes', 'symbols'))
|
||
_DROP = object()
|
||
|
||
|
||
def normalize(value, *, prefix=False):
|
||
"""前缀、点式、六位裸码 → 唯一代码;北交所、错市场、非法值返回 None。"""
|
||
if value is None or isinstance(value, bool):
|
||
return None
|
||
hit = _CODE.fullmatch(str(value).strip().upper())
|
||
if not hit:
|
||
return None
|
||
explicit = hit.group(1) or hit.group(4)
|
||
number = hit.group(2) or hit.group(3) or hit.group(5)
|
||
market = 'SH' if number.startswith(('60', '68')) else 'SZ' if number.startswith(('00', '30')) else None
|
||
if market is None or (explicit and explicit != market):
|
||
return None
|
||
return market + number if prefix else number + '.' + market
|
||
|
||
|
||
def sql_filter(column, *, postgres=False):
|
||
"""仅接受本地列名,SQL 在读取前排除范围外证券;内存仍二次核验。"""
|
||
if not re.fullmatch(r'(?:[A-Za-z_][A-Za-z0-9_]*[.])?`?[A-Za-z_][A-Za-z0-9_]*`?', column):
|
||
raise ValueError('证券代码列名非法')
|
||
cast = 'TEXT' if postgres else 'CHAR'
|
||
op = '~' if postgres else 'REGEXP'
|
||
return f"UPPER(TRIM(CAST({column} AS {cast}))) {op} '{_SQL_CODE}'"
|
||
|
||
|
||
def frame(df, column, *, prefix=False):
|
||
"""样本入口过滤后再计算;不修改调用方 DataFrame。"""
|
||
if df is None or df.empty:
|
||
return df
|
||
if column not in df.columns:
|
||
raise ValueError('证券样本缺少代码列:' + column)
|
||
codes = df[column].map(lambda value: normalize(value, prefix=prefix))
|
||
result = df.loc[codes.notna()].copy()
|
||
result[column] = codes.loc[codes.notna()]
|
||
return result
|
||
|
||
|
||
def _nested(value, *, members=False):
|
||
encoded = isinstance(value, str) and value.lstrip().startswith(('[', '{'))
|
||
if encoded:
|
||
try:
|
||
value = json.loads(value)
|
||
except ValueError:
|
||
if members:
|
||
raise ValueError("冻结成员不是合法 JSON") from None
|
||
return value
|
||
if isinstance(value, dict):
|
||
code_fields = [key for key in _CODE_FIELDS if key in value]
|
||
if code_fields:
|
||
codes = [normalize(value[key]) for key in code_fields]
|
||
if None in codes or len(set(codes)) != 1:
|
||
return _DROP
|
||
elif members:
|
||
return _DROP # 未能确认证券身份的成员不新增归档。
|
||
out = {}
|
||
for key, val in value.items():
|
||
if key in code_fields:
|
||
out[key] = normalize(val, prefix=key == 'stock_code')
|
||
continue
|
||
clean = _nested(val, members=key in _MEMBER_FIELDS)
|
||
if clean is not _DROP:
|
||
out[key] = clean
|
||
elif isinstance(value, (list, tuple)):
|
||
out = []
|
||
for item in value:
|
||
clean = _nested(item, members=members)
|
||
if clean is not _DROP:
|
||
out.append(clean)
|
||
elif members:
|
||
out = normalize(value)
|
||
if out is None:
|
||
return _DROP
|
||
else:
|
||
out = value
|
||
return json.dumps(out, ensure_ascii=False) if encoded else out
|
||
|
||
|
||
def frozen_frame(df):
|
||
"""过滤冻结中的证券行与嵌套成员;行业关系等非证券行保持。"""
|
||
if df is None or df.empty:
|
||
return df
|
||
out = df.copy()
|
||
for column in _CODE_FIELDS:
|
||
if column in out.columns:
|
||
out = frame(out, column, prefix=column == 'stock_code')
|
||
for column in out.columns:
|
||
if column in _CODE_FIELDS:
|
||
continue
|
||
def clean(value):
|
||
result = _nested(value, members=column in _MEMBER_FIELDS)
|
||
return None if result is _DROP else result
|
||
out[column] = out[column].map(clean)
|
||
return out
|