109 lines
4.2 KiB
Python
109 lines
4.2 KiB
Python
|
|
"""沪深 A 股样本边界;只规范化代码,不推断上市资格或补造资料。"""
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
import re
|
|||
|
|
|
|||
|
|
# 与项目证券入口一致:沪市 60/68、深市 00/30;显式市场必须与号段相符。
|
|||
|
|
_CODE = re.compile(r'^(?:(SH|SZ)([0-9]{6})|([0-9]{6})[.](SH|SZ)|([0-9]{6}))$')
|
|||
|
|
_SQL_CODE = ('^(SH(60|68)[0-9]{4}|(60|68)[0-9]{4}[.]SH|(60|68)[0-9]{4}'
|
|||
|
|
'|SZ(00|30)[0-9]{4}|(00|30)[0-9]{4}[.]SZ|(00|30)[0-9]{4})$')
|
|||
|
|
_CODE_FIELDS = ('ts_code', 'stock_code', 'symbol', 'code')
|
|||
|
|
_MEMBER_FIELDS = frozenset(('members', 'quiet', 'moved_members', 'constituents', 'stocks',
|
|||
|
|
'ts_codes', 'stock_codes', 'symbols'))
|
|||
|
|
_DROP = object()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def normalize(value, *, prefix=False):
|
|||
|
|
"""前缀、点式、六位裸码 → 唯一代码;北交所、错市场、非法值返回 None。"""
|
|||
|
|
if value is None or isinstance(value, bool):
|
|||
|
|
return None
|
|||
|
|
hit = _CODE.fullmatch(str(value).strip().upper())
|
|||
|
|
if not hit:
|
|||
|
|
return None
|
|||
|
|
explicit = hit.group(1) or hit.group(4)
|
|||
|
|
number = hit.group(2) or hit.group(3) or hit.group(5)
|
|||
|
|
market = 'SH' if number.startswith(('60', '68')) else 'SZ' if number.startswith(('00', '30')) else None
|
|||
|
|
if market is None or (explicit and explicit != market):
|
|||
|
|
return None
|
|||
|
|
return market + number if prefix else number + '.' + market
|
|||
|
|
|
|||
|
|
|
|||
|
|
def sql_filter(column, *, postgres=False):
|
|||
|
|
"""仅接受本地列名,SQL 在读取前排除范围外证券;内存仍二次核验。"""
|
|||
|
|
if not re.fullmatch(r'(?:[A-Za-z_][A-Za-z0-9_]*[.])?`?[A-Za-z_][A-Za-z0-9_]*`?', column):
|
|||
|
|
raise ValueError('证券代码列名非法')
|
|||
|
|
cast = 'TEXT' if postgres else 'CHAR'
|
|||
|
|
op = '~' if postgres else 'REGEXP'
|
|||
|
|
return f"UPPER(TRIM(CAST({column} AS {cast}))) {op} '{_SQL_CODE}'"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def frame(df, column, *, prefix=False):
|
|||
|
|
"""样本入口过滤后再计算;不修改调用方 DataFrame。"""
|
|||
|
|
if df is None or df.empty:
|
|||
|
|
return df
|
|||
|
|
if column not in df.columns:
|
|||
|
|
raise ValueError('证券样本缺少代码列:' + column)
|
|||
|
|
codes = df[column].map(lambda value: normalize(value, prefix=prefix))
|
|||
|
|
result = df.loc[codes.notna()].copy()
|
|||
|
|
result[column] = codes.loc[codes.notna()]
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _nested(value, *, members=False):
|
|||
|
|
encoded = isinstance(value, str) and value.lstrip().startswith(('[', '{'))
|
|||
|
|
if encoded:
|
|||
|
|
try:
|
|||
|
|
value = json.loads(value)
|
|||
|
|
except ValueError:
|
|||
|
|
if members:
|
|||
|
|
raise ValueError("冻结成员不是合法 JSON") from None
|
|||
|
|
return value
|
|||
|
|
if isinstance(value, dict):
|
|||
|
|
code_fields = [key for key in _CODE_FIELDS if key in value]
|
|||
|
|
if code_fields:
|
|||
|
|
codes = [normalize(value[key]) for key in code_fields]
|
|||
|
|
if None in codes or len(set(codes)) != 1:
|
|||
|
|
return _DROP
|
|||
|
|
elif members:
|
|||
|
|
return _DROP # 未能确认证券身份的成员不新增归档。
|
|||
|
|
out = {}
|
|||
|
|
for key, val in value.items():
|
|||
|
|
if key in code_fields:
|
|||
|
|
out[key] = normalize(val, prefix=key == 'stock_code')
|
|||
|
|
continue
|
|||
|
|
clean = _nested(val, members=key in _MEMBER_FIELDS)
|
|||
|
|
if clean is not _DROP:
|
|||
|
|
out[key] = clean
|
|||
|
|
elif isinstance(value, (list, tuple)):
|
|||
|
|
out = []
|
|||
|
|
for item in value:
|
|||
|
|
clean = _nested(item, members=members)
|
|||
|
|
if clean is not _DROP:
|
|||
|
|
out.append(clean)
|
|||
|
|
elif members:
|
|||
|
|
out = normalize(value)
|
|||
|
|
if out is None:
|
|||
|
|
return _DROP
|
|||
|
|
else:
|
|||
|
|
out = value
|
|||
|
|
return json.dumps(out, ensure_ascii=False) if encoded else out
|
|||
|
|
|
|||
|
|
|
|||
|
|
def frozen_frame(df):
|
|||
|
|
"""过滤冻结中的证券行与嵌套成员;行业关系等非证券行保持。"""
|
|||
|
|
if df is None or df.empty:
|
|||
|
|
return df
|
|||
|
|
out = df.copy()
|
|||
|
|
for column in _CODE_FIELDS:
|
|||
|
|
if column in out.columns:
|
|||
|
|
out = frame(out, column, prefix=column == 'stock_code')
|
|||
|
|
for column in out.columns:
|
|||
|
|
if column in _CODE_FIELDS:
|
|||
|
|
continue
|
|||
|
|
def clean(value):
|
|||
|
|
result = _nested(value, members=column in _MEMBER_FIELDS)
|
|||
|
|
return None if result is _DROP else result
|
|||
|
|
out[column] = out[column].map(clean)
|
|||
|
|
return out
|