akg-factor-bridge/plan_versions.py

269 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""只追加的计划版本。目录内文件锁涵盖采集、校验与发布,最新文件原子替换。"""
from __future__ import annotations
import contextlib
import copy
import datetime as dt
import fcntl
import hashlib
import json
import os
from pathlib import Path
import re
import tempfile
def digest(value):
return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True,
separators=(',', ':'), allow_nan=False).encode()).hexdigest()
def now():
return dt.datetime.now(dt.timezone.utc).isoformat(timespec='seconds')
def _date(day):
return dt.date.fromisoformat(day).isoformat()
def atomic(path, value, *, text=False):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(prefix='.' + path.name, dir=path.parent)
try:
with os.fdopen(fd, 'w', encoding='utf-8') as f:
if text:
f.write(value)
else:
json.dump(value, f, ensure_ascii=False, sort_keys=True, allow_nan=False)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, path)
directory = os.open(str(path.parent), os.O_RDONLY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
if os.path.exists(tmp):
os.unlink(tmp)
@contextlib.contextmanager
def writer(root):
Path(root).mkdir(parents=True, exist_ok=True)
with open(Path(root) / '.publish.lock', 'a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
yield
def latest_date(root):
days = sorted(p.stem[5:] for p in Path(root).glob('plan_????-??-??.json'))
return days[-1] if days else None
def load(root, day=None, decision_id=None, morning=False):
if decision_id:
if not re.fullmatch(r'\d{4}-\d{2}-\d{2}_[0-9a-f]{64}', decision_id):
raise ValueError('计划版本标识无效')
if day and day != decision_id[:10]:
raise ValueError('计划日期与版本不匹配')
latest = load(root, decision_id[:10])
d = latest.get('decision') or {}
if 'ancestor_ids' not in d:
for published in history(root, decision_id[:10]):
if published['decision']['id'] == decision_id:
return published
raise FileNotFoundError('该版本尚未发布')
if decision_id != d.get('id') and decision_id not in d['ancestor_ids']:
raise FileNotFoundError('该版本尚未发布')
path = Path(root) / 'versions' / decision_id[:10] / (decision_id + '.json')
result = json.loads(path.read_text())['plan']
if result.get('decision', {}).get('id') != decision_id:
raise ValueError('归档版本身份不一致')
if decision_id == d.get('id') and result != latest:
raise ValueError('发布入口与不可变版本不一致')
return result
day = _date(day or latest_date(root) or '')
if morning:
pointer = Path(root) / 'versions' / day / 'morning.json'
if not pointer.exists():
# 最新指针切换成功、晨版指针写入前中断:沿已发布父链恢复首版。
versions = history(root, day)
if not versions:
raise FileNotFoundError('无可核验晨版')
return versions[0]
return load(root, day, json.loads(pointer.read_text())['id'])
return json.loads((Path(root) / f'plan_{day}.json').read_text())
def envelope(root, decision_id):
load(root, decision_id=decision_id) # 校验标识,不接受任意文件路径
return json.loads((Path(root) / 'versions' / decision_id[:10] / (decision_id + '.json')).read_text())
def history(root, day):
"""只沿已发布父链,目录里未切换成功的文件不是发布历史。"""
current = load(root, day)
result, seen = [], set()
while current.get('decision'):
d = current['decision']
if not re.fullmatch(re.escape(day) + r'_[0-9a-f]{64}', d['id']):
raise ValueError('计划版本标识无效')
archived = json.loads((Path(root) / 'versions' / day / (d['id'] + '.json')).read_text())['plan']
if archived != current:
raise ValueError('发布入口与不可变版本不一致')
if d['id'] in seen:
raise ValueError('计划版本父链成环')
seen.add(d['id'])
result.append(archived)
if not d.get('parent_id'):
break
parent = d['parent_id']
if not re.fullmatch(re.escape(day) + r'_[0-9a-f]{64}', parent):
raise ValueError('计划父版本标识无效')
current = json.loads((Path(root) / 'versions' / day / (parent + '.json')).read_text())['plan']
return list(reversed(result))
def health(root, day, **values):
path = Path(root) / 'versions' / _date(day) / 'health.json'
if values:
atomic(path, {'checked_at': now(), **values})
return json.loads(path.read_text()) if path.exists() else {}
def morning_eligible(plan):
"""行情日后首个交易日开盘前才是晨版;盘中补发不冒充晨版历史。"""
from zoneinfo import ZoneInfo
from chinese_calendar import is_holiday
try:
day = dt.date.fromisoformat(plan['date']) + dt.timedelta(days=1)
while day.weekday() >= 5 or is_holiday(day):
day += dt.timedelta(days=1)
cutoff = dt.datetime.fromisoformat(plan['decision']['cutoff_at'])
local = cutoff.astimezone(ZoneInfo('Asia/Shanghai'))
return (cutoff.tzinfo is not None and local.date() == day
and local.time().replace(tzinfo=None) < dt.time(9, 30))
except (KeyError, TypeError, ValueError):
return False
def research(cr, previous=None):
cr = cr or {}
previous = previous or {}
keys = ('overall', 'valuation', 'doubt_hard', 'expectation', 'cycle', 'period_fi',
'period_rf', 'n_materials', 'unknown_n', 'numbers_unmatched', 'thesis', 'antithesis')
return {k: copy.deepcopy(cr.get(k)) for k in keys} | {
'confidence': cr.get('confidence'), 'review_ref': cr.get('review_ref'),
'changes': [{'field': k, 'before': previous.get(k), 'after': cr.get(k)}
for k in keys if previous and previous.get(k) != cr.get(k)],
'checks': [{'kind': k, 'text': cr.get(k), 'status': '待核验'}
for k in ('catalyst', 'invalidation') if cr.get(k)],
'report_ran_at': cr.get('report_ran_at') or cr.get('ran_at'),
'material_status': cr.get('qual'),
'report_version_url': cr.get('report_version_url'),
}
def publish(root, plan, inputs, params, *, cutoff_at, reason):
"""调用方必须持 writer。指纹包含代码、固定输入及配置;时间不导致空版本。"""
day = _date(plan['date'])
old = None
try:
old = load(root, day)
except FileNotFoundError:
pass
if old is not None and not old.get('decision'):
raise ValueError('旧格式计划只读保留,禁止以当前输入替换历史')
ih, ph = digest(inputs), digest(params)
prior = (old or {}).get('decision') or {}
if (prior.get('input_hash') == ih and prior.get('params_hash') == ph
and old.get('plan_version') == plan.get('plan_version')):
health(root, day, pending_codes=[], error=None, decision_id=prior['id'])
return old
identity = digest({'input': ih, 'params': ph, 'code': plan.get('plan_version'),
'date': day, 'parent': prior.get('id')})
did = day + '_' + identity
if (old or {}).get('decision', {}).get('id') == did:
return old
p = copy.deepcopy(plan)
prev = {r['code']: r for bucket in ('main', 'observe') for r in (old or {}).get(bucket, [])}
by_code = {}
rank = 0
for bucket in ('main', 'observe'):
for row in p.get(bucket, []):
rank += 1
if row['code'] in by_code:
raise ValueError('完整计划出现重复股票: ' + row['code'])
row['research_rank'] = rank
row['review_ref'] = (row.get('company_review') or {}).get('review_ref')
row['research'] = research(row.get('company_review'), (prev.get(row['code']) or {}).get('company_review'))
by_code[row['code']] = row
for bucket in ('candidates', 'watch', 'main_shown', 'observe_shown'):
p[bucket] = [{**copy.deepcopy(by_code[r['code']]), 'rank': r.get('rank', i)}
for i, r in enumerate(p.get(bucket, []), 1)]
p['schema_version'] = 2
ancestors = prior.get('ancestor_ids')
if prior and ancestors is None:
ancestors = [v['decision']['id'] for v in history(root, day)[:-1]]
p['decision'] = {'id': did, 'parent_id': (old or {}).get('decision', {}).get('id'),
'ancestor_ids': ((ancestors or []) + [prior['id']]) if prior else [],
'cutoff_at': cutoff_at, 'input_hash': ih, 'params_hash': ph, 'reason': reason}
p['decision']['publication_kind'] = ('revision' if prior else
'morning' if morning_eligible(p) else 'recovery')
if not prior and p['decision']['publication_kind'] == 'recovery':
p['decision']['reason'] = '数据恢复后补发'
p['generated_at'] = now()
p['traceability'] = 'captured_inputs'
# 返回与落盘同一 JSON 形状(元组等不得在内存出口与历史出口有两种表示)。
p = json.loads(json.dumps(p, ensure_ascii=False, allow_nan=False))
version_dir = Path(root) / 'versions' / day
path = version_dir / (did + '.json')
if path.exists():
# 中断重试恢复已经落盘但尚未切换的同一版本。
p = json.loads(path.read_text())['plan']
else:
atomic(path, {'plan': p, 'inputs': inputs, 'params': params})
morning = version_dir / 'morning.json'
atomic(Path(root) / f'plan_{day}.json', p)
if not morning.exists():
atomic(morning, {'id': history(root, day)[0]['decision']['id']})
health(root, day, pending_codes=[], error=None, decision_id=did)
return p
def stock_check(root, code):
"""只读完整发布文件的一只股票。不存在与取数失败严格区分;不按展示名次裁剪。"""
if not re.fullmatch(r'(SH|SZ|BJ)\d{6}', code):
raise ValueError('股票代码无效')
for _ in range(3):
day = latest_date(root)
if not day:
raise FileNotFoundError('无已发布计划')
path = Path(root) / f'plan_{day}.json'
health_path = Path(root) / 'versions' / day / 'health.json'
def identity(p):
try:
s = p.stat()
return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
except FileNotFoundError:
return None
before = (identity(path), identity(health_path))
published = load(root, day)
state = health(root, day)
if (day != latest_date(root) or before != (identity(path), identity(health_path))):
continue
if (published.get('date') != day or not (published.get('decision') or {}).get('id')
or not all(isinstance(published.get(b), list) for b in ('main', 'observe'))):
raise ValueError('完整发布计划不可核验')
rows = {b: [r for r in published[b] if r.get('code') == code] for b in ('main', 'observe')}
count = sum(map(len, rows.values()))
if count > 1:
raise ValueError('完整计划出现重复股票')
return {**{k: published.get(k) for k in ('date', 'schema_version', 'decision', 'traceability',
'plan_version', 'generated_at', 'rank_axis', 'gate_on')},
**rows, 'check_code': code, 'found': bool(count), 'refresh_status': state,
'plan_source': 'snapshot', 'snapshot_generated_at': published.get('generated_at')}
raise RuntimeError('计划正在切换,请稍后重试')