akg-factor-bridge/plan_inputs.py

162 lines
6.1 KiB
Python

"""计划输入的录制与重放。仅捕获显式来源边界,重放绝不回退到数据库。"""
from __future__ import annotations
import contextlib
import contextvars
import datetime as dt
import decimal
import hashlib
import json
import math
import re
from pathlib import Path
_active = contextvars.ContextVar('plan_inputs', default=None)
def pack(v):
import pandas as pd
if v is pd.NA or v is pd.NaT:
return None
if isinstance(v, pd.DataFrame):
return {'$frame': pack(v.to_dict('split')), 'dtypes': {str(k): str(t) for k, t in v.dtypes.items()}}
if isinstance(v, pd.Series):
return {'$series': pack({'index': list(v.index), 'data': list(v), 'name': v.name}), 'dtype': str(v.dtype)}
if isinstance(v, dict):
return {'$dict': [[pack(k), pack(x)] for k, x in sorted(v.items(), key=lambda p: str(p[0]))]}
if isinstance(v, (list, tuple, set)):
xs = sorted(v, key=str) if isinstance(v, set) else v
return {'$' + type(v).__name__: [pack(x) for x in xs]}
if isinstance(v, dt.datetime):
return {'$datetime': v.isoformat()}
if isinstance(v, dt.date):
return {'$date': v.isoformat()}
if isinstance(v, decimal.Decimal):
return {'$decimal': str(v)}
if isinstance(v, float) and not math.isfinite(v):
return {'$float': str(v)}
if v is None or isinstance(v, (str, int, float, bool)):
return v
if hasattr(v, 'item'):
return pack(v.item())
raise TypeError(f'计划输入不能序列化: {type(v).__name__}')
def unpack(v):
if not isinstance(v, dict):
return v
if '$frame' in v:
import pandas as pd
p = unpack(v['$frame'])
out = pd.DataFrame(p['data'], columns=p['columns'], index=p['index'])
for k, dtype in v.get('dtypes', {}).items():
if dtype != 'object' and k in out:
out[k] = out[k].astype(dtype)
return out
if '$series' in v:
import pandas as pd
p = unpack(v['$series'])
return pd.Series(p['data'], index=p['index'], name=p['name'], dtype=v['dtype'])
if '$dict' in v:
return {unpack(k): unpack(x) for k, x in v['$dict']}
for kind, ctor in (('list', list), ('tuple', tuple), ('set', set)):
if '$' + kind in v:
return ctor(unpack(x) for x in v['$' + kind])
for kind, ctor in (('date', dt.date.fromisoformat), ('datetime', dt.datetime.fromisoformat),
('decimal', decimal.Decimal), ('float', float)):
if '$' + kind in v:
return ctor(v['$' + kind])
raise ValueError('未知计划输入编码')
def digest(v):
return hashlib.sha256(json.dumps(v, ensure_ascii=False, sort_keys=True,
separators=(',', ':'), allow_nan=False).encode()).hexdigest()
def effective_config(module):
names = set()
for name in ('plan.py', 'card.py', 'logic_state_daily.py'):
names.update(re.findall(r'config\.([A-Z][A-Z_0-9]+)', Path(__file__).with_name(name).read_text()))
return {k: getattr(module, k) for k in sorted(names) if hasattr(module, k)
and k not in {'PLAN_SNAPSHOT_DIR'}}
class Bundle:
def __init__(self, *, records=None, params=None, cutoff_at=None):
self.replay = records is not None
self.records = {} if records is None else dict(records)
self.params = params or {}
self.cutoff_at = cutoff_at or dt.datetime.now(dt.timezone.utc).isoformat()
self.missing = []
@contextlib.contextmanager
def use(self):
token = _active.set(self)
try:
yield self
if self.missing:
raise RuntimeError('冻结输入缺失,禁止回源: ' + ', '.join(self.missing))
finally:
_active.reset(token)
def call(self, name, fn, args, kw):
key = name + ':' + digest(pack([args, kw]))
if key in self.records:
rec = self.records[key]
if 'error' in rec:
raise RuntimeError(rec['error'])
return unpack(rec['value'])
if self.replay:
self.missing.append(name)
raise RuntimeError('冻结输入缺失: ' + name)
try:
actual = dict(kw)
if name == 'sources.company_reviews':
import config
actual.update(cutoff_at=self.cutoff_at, strict=True,
include_rating=self.params.get('PLAN_RATING_PRIORITY_ENABLED', False) if self.replay else
self.params.get('PLAN_RATING_PRIORITY_ENABLED', config.PLAN_RATING_PRIORITY_ENABLED),
rating_days=self.params.get('PLAN_RATING_PRIORITY_DAYS', 30))
result = fn(*args, **actual)
except Exception as e:
self.records[key] = {'name': name, 'error': type(e).__name__, 'args': pack([args, kw])}
raise
self.records[key] = {'name': name, 'args': pack([args, kw]), 'value': pack(result)}
return unpack(self.records[key]['value'])
def read(name, fn, *args, **kw):
bundle = _active.get()
return bundle.call(name, fn, args, kw) if bundle is not None else fn(*args, **kw)
class Readers:
def __init__(self, module, names=None):
self.module, self.names = module, names
def __getattr__(self, name):
fn = getattr(self.module, name)
if not callable(fn) or (self.names is not None and name not in self.names):
return fn
return lambda *a, **kw: read(self.module.__name__ + '.' + name, fn, *a, **kw)
class Config:
def __init__(self, module):
self.module = module
def __getattr__(self, name):
bundle = _active.get()
if bundle is not None and name in bundle.params:
return bundle.params[name]
# 老冻结版本从未启用新规则,不能继承今天服务器的开关。
if bundle is not None and bundle.replay and name in ('PLAN_RATING_PRIORITY_ENABLED', 'CARD_CONSENSUS_ADMISSION_ENABLED'):
return False
return getattr(self.module, name)
def factor_read(name, *args):
import factors
return read('factors.' + name, getattr(factors, name), *args)