273 lines
12 KiB
Python
273 lines
12 KiB
Python
"""只追加的计划版本。目录内文件锁涵盖采集、校验与发布,最新文件原子替换。"""
|
||
from __future__ import annotations
|
||
|
||
import contextlib
|
||
import copy
|
||
import datetime as dt
|
||
import fcntl
|
||
import hashlib
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import tempfile
|
||
|
||
|
||
def digest(value):
|
||
return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True,
|
||
separators=(',', ':'), allow_nan=False).encode()).hexdigest()
|
||
|
||
|
||
def now():
|
||
return dt.datetime.now(dt.timezone.utc).isoformat(timespec='seconds')
|
||
|
||
|
||
def _date(day):
|
||
return dt.date.fromisoformat(day).isoformat()
|
||
|
||
|
||
def atomic(path, value, *, text=False):
|
||
path = Path(path)
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
fd, tmp = tempfile.mkstemp(prefix='.' + path.name, dir=path.parent)
|
||
try:
|
||
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
||
if text:
|
||
f.write(value)
|
||
else:
|
||
json.dump(value, f, ensure_ascii=False, sort_keys=True, allow_nan=False)
|
||
f.flush()
|
||
os.fsync(f.fileno())
|
||
os.replace(tmp, path)
|
||
directory = os.open(str(path.parent), os.O_RDONLY)
|
||
try:
|
||
os.fsync(directory)
|
||
finally:
|
||
os.close(directory)
|
||
finally:
|
||
if os.path.exists(tmp):
|
||
os.unlink(tmp)
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def writer(root):
|
||
Path(root).mkdir(parents=True, exist_ok=True)
|
||
with open(Path(root) / '.publish.lock', 'a') as lock:
|
||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||
yield
|
||
|
||
|
||
def latest_date(root):
|
||
days = sorted(p.stem[5:] for p in Path(root).glob('plan_????-??-??.json'))
|
||
return days[-1] if days else None
|
||
|
||
|
||
def load(root, day=None, decision_id=None, morning=False):
|
||
if decision_id:
|
||
if not re.fullmatch(r'\d{4}-\d{2}-\d{2}_[0-9a-f]{64}', decision_id):
|
||
raise ValueError('计划版本标识无效')
|
||
if day and day != decision_id[:10]:
|
||
raise ValueError('计划日期与版本不匹配')
|
||
latest = load(root, decision_id[:10])
|
||
d = latest.get('decision') or {}
|
||
if 'ancestor_ids' not in d:
|
||
for published in history(root, decision_id[:10]):
|
||
if published['decision']['id'] == decision_id:
|
||
return published
|
||
raise FileNotFoundError('该版本尚未发布')
|
||
if decision_id != d.get('id') and decision_id not in d['ancestor_ids']:
|
||
raise FileNotFoundError('该版本尚未发布')
|
||
path = Path(root) / 'versions' / decision_id[:10] / (decision_id + '.json')
|
||
result = json.loads(path.read_text())['plan']
|
||
if result.get('decision', {}).get('id') != decision_id:
|
||
raise ValueError('归档版本身份不一致')
|
||
if decision_id == d.get('id') and result != latest:
|
||
raise ValueError('发布入口与不可变版本不一致')
|
||
return result
|
||
day = _date(day or latest_date(root) or '')
|
||
if morning:
|
||
pointer = Path(root) / 'versions' / day / 'morning.json'
|
||
if not pointer.exists():
|
||
# 最新指针切换成功、晨版指针写入前中断:沿已发布父链恢复首版。
|
||
versions = history(root, day)
|
||
if not versions:
|
||
raise FileNotFoundError('无可核验晨版')
|
||
return versions[0]
|
||
return load(root, day, json.loads(pointer.read_text())['id'])
|
||
return json.loads((Path(root) / f'plan_{day}.json').read_text())
|
||
|
||
|
||
def envelope(root, decision_id):
|
||
load(root, decision_id=decision_id) # 校验标识,不接受任意文件路径
|
||
return json.loads((Path(root) / 'versions' / decision_id[:10] / (decision_id + '.json')).read_text())
|
||
|
||
|
||
def history(root, day):
|
||
"""只沿已发布父链,目录里未切换成功的文件不是发布历史。"""
|
||
current = load(root, day)
|
||
result, seen = [], set()
|
||
while current.get('decision'):
|
||
d = current['decision']
|
||
if not re.fullmatch(re.escape(day) + r'_[0-9a-f]{64}', d['id']):
|
||
raise ValueError('计划版本标识无效')
|
||
archived = json.loads((Path(root) / 'versions' / day / (d['id'] + '.json')).read_text())['plan']
|
||
if archived != current:
|
||
raise ValueError('发布入口与不可变版本不一致')
|
||
if d['id'] in seen:
|
||
raise ValueError('计划版本父链成环')
|
||
seen.add(d['id'])
|
||
result.append(archived)
|
||
if not d.get('parent_id'):
|
||
break
|
||
parent = d['parent_id']
|
||
if not re.fullmatch(re.escape(day) + r'_[0-9a-f]{64}', parent):
|
||
raise ValueError('计划父版本标识无效')
|
||
current = json.loads((Path(root) / 'versions' / day / (parent + '.json')).read_text())['plan']
|
||
return list(reversed(result))
|
||
|
||
|
||
def health(root, day, **values):
|
||
path = Path(root) / 'versions' / _date(day) / 'health.json'
|
||
if values:
|
||
atomic(path, {'checked_at': now(), **values})
|
||
return json.loads(path.read_text()) if path.exists() else {}
|
||
|
||
|
||
def morning_eligible(plan):
|
||
"""行情日后首个交易日开盘前才是晨版;盘中补发不冒充晨版历史。"""
|
||
from zoneinfo import ZoneInfo
|
||
from chinese_calendar import is_holiday
|
||
try:
|
||
day = dt.date.fromisoformat(plan['date']) + dt.timedelta(days=1)
|
||
while day.weekday() >= 5 or is_holiday(day):
|
||
day += dt.timedelta(days=1)
|
||
cutoff = dt.datetime.fromisoformat(plan['decision']['cutoff_at'])
|
||
local = cutoff.astimezone(ZoneInfo('Asia/Shanghai'))
|
||
return (cutoff.tzinfo is not None and local.date() == day
|
||
and local.time().replace(tzinfo=None) < dt.time(9, 30))
|
||
except (KeyError, TypeError, ValueError):
|
||
return False
|
||
|
||
|
||
def research(cr, previous=None):
|
||
cr = cr or {}
|
||
previous = previous or {}
|
||
keys = ('overall', 'valuation', 'doubt_hard', 'expectation', 'cycle', 'period_fi',
|
||
'period_rf', 'n_materials', 'unknown_n', 'numbers_unmatched', 'thesis', 'antithesis')
|
||
return {k: copy.deepcopy(cr.get(k)) for k in keys} | {
|
||
'confidence': cr.get('confidence'), 'review_ref': cr.get('review_ref'),
|
||
'changes': [{'field': k, 'before': previous.get(k), 'after': cr.get(k)}
|
||
for k in keys if previous and previous.get(k) != cr.get(k)],
|
||
'checks': [{'kind': k, 'text': cr.get(k), 'status': '待核验'}
|
||
for k in ('catalyst', 'invalidation') if cr.get(k)],
|
||
'report_ran_at': cr.get('report_ran_at') or cr.get('ran_at'),
|
||
'material_status': cr.get('qual'),
|
||
'report_version_url': cr.get('report_version_url'),
|
||
}
|
||
|
||
|
||
def publish(root, plan, inputs, params, *, cutoff_at, reason):
|
||
"""调用方必须持 writer。指纹包含代码、固定输入及配置;时间不导致空版本。"""
|
||
day = _date(plan['date'])
|
||
old = None
|
||
try:
|
||
old = load(root, day)
|
||
except FileNotFoundError:
|
||
pass
|
||
if old is not None and not old.get('decision'):
|
||
raise ValueError('旧格式计划只读保留,禁止以当前输入替换历史')
|
||
ih, ph = digest(inputs), digest(params)
|
||
prior = (old or {}).get('decision') or {}
|
||
if (prior.get('input_hash') == ih and prior.get('params_hash') == ph
|
||
and old.get('plan_version') == plan.get('plan_version')):
|
||
health(root, day, pending_codes=[], error=None, decision_id=prior['id'])
|
||
return old
|
||
identity = digest({'input': ih, 'params': ph, 'code': plan.get('plan_version'),
|
||
'date': day, 'parent': prior.get('id')})
|
||
did = day + '_' + identity
|
||
if (old or {}).get('decision', {}).get('id') == did:
|
||
return old
|
||
p = copy.deepcopy(plan)
|
||
prev = {r['code']: r for bucket in ('main', 'observe') for r in (old or {}).get(bucket, [])}
|
||
by_code = {}
|
||
rank = 0
|
||
for bucket in ('main', 'observe'):
|
||
for row in p.get(bucket, []):
|
||
rank += 1
|
||
if row['code'] in by_code:
|
||
raise ValueError('完整计划出现重复股票: ' + row['code'])
|
||
row['research_rank'] = rank
|
||
row['review_ref'] = (row.get('company_review') or {}).get('review_ref')
|
||
row['research'] = research(row.get('company_review'), (prev.get(row['code']) or {}).get('company_review'))
|
||
by_code[row['code']] = row
|
||
for bucket in ('candidates', 'watch', 'main_shown', 'observe_shown'):
|
||
p[bucket] = [{**copy.deepcopy(by_code[r['code']]), 'rank': r.get('rank', i)}
|
||
for i, r in enumerate(p.get(bucket, []), 1)]
|
||
p['schema_version'] = 2
|
||
ancestors = prior.get('ancestor_ids')
|
||
if prior and ancestors is None:
|
||
ancestors = [v['decision']['id'] for v in history(root, day)[:-1]]
|
||
p['decision'] = {'id': did, 'parent_id': (old or {}).get('decision', {}).get('id'),
|
||
'ancestor_ids': ((ancestors or []) + [prior['id']]) if prior else [],
|
||
'cutoff_at': cutoff_at, 'input_hash': ih, 'params_hash': ph, 'reason': reason}
|
||
p['decision']['publication_kind'] = ('revision' if prior else
|
||
'morning' if morning_eligible(p) else 'recovery')
|
||
if not prior and p['decision']['publication_kind'] == 'recovery':
|
||
p['decision']['reason'] = '数据恢复后补发'
|
||
p['generated_at'] = now()
|
||
p['traceability'] = 'captured_inputs'
|
||
# 返回与落盘同一 JSON 形状(元组等不得在内存出口与历史出口有两种表示)。
|
||
p = json.loads(json.dumps(p, ensure_ascii=False, allow_nan=False))
|
||
version_dir = Path(root) / 'versions' / day
|
||
path = version_dir / (did + '.json')
|
||
if path.exists():
|
||
# 中断重试恢复已经落盘但尚未切换的同一版本。
|
||
p = json.loads(path.read_text())['plan']
|
||
else:
|
||
atomic(path, {'plan': p, 'inputs': inputs, 'params': params})
|
||
morning = version_dir / 'morning.json'
|
||
atomic(Path(root) / f'plan_{day}.json', p)
|
||
if not morning.exists():
|
||
atomic(morning, {'id': history(root, day)[0]['decision']['id']})
|
||
health(root, day, pending_codes=[], error=None, decision_id=did)
|
||
return p
|
||
|
||
|
||
def stock_check(root, code, include_user_candidates=False):
|
||
"""只读完整发布文件的一只股票。不存在与取数失败严格区分;不按展示名次裁剪。"""
|
||
if not re.fullmatch(r'(SH|SZ|BJ)\d{6}', code):
|
||
raise ValueError('股票代码无效')
|
||
for _ in range(3):
|
||
day = latest_date(root)
|
||
if not day:
|
||
raise FileNotFoundError('无已发布计划')
|
||
path = Path(root) / f'plan_{day}.json'
|
||
health_path = Path(root) / 'versions' / day / 'health.json'
|
||
candidates_path = Path(root) / 'user_candidates.json'
|
||
def identity(p):
|
||
try:
|
||
s = p.stat()
|
||
return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
|
||
except FileNotFoundError:
|
||
return None
|
||
before = (identity(path), identity(health_path), identity(candidates_path))
|
||
published = load(root, day)
|
||
state = health(root, day)
|
||
import user_candidates as uc
|
||
state = dict(state, pending_codes=sorted(set(state.get('pending_codes') or []) | set(uc.live_pending(root, published))))
|
||
if (day != latest_date(root) or before != (identity(path), identity(health_path), identity(candidates_path))):
|
||
continue
|
||
if (published.get('date') != day or not (published.get('decision') or {}).get('id')
|
||
or not all(isinstance(published.get(b), list) for b in ('main', 'observe'))):
|
||
raise ValueError('完整发布计划不可核验')
|
||
published = uc.consumer_view(published, include_user_candidates)
|
||
rows = {b: [r for r in published[b] if r.get('code') == code] for b in ('main', 'observe')}
|
||
count = sum(map(len, rows.values()))
|
||
if count > 1:
|
||
raise ValueError('完整计划出现重复股票')
|
||
return {**{k: published.get(k) for k in ('date', 'schema_version', 'decision', 'traceability',
|
||
'plan_version', 'generated_at', 'rank_axis', 'gate_on')},
|
||
**rows, 'check_code': code, 'found': bool(count), 'refresh_status': state,
|
||
'plan_source': 'snapshot', 'snapshot_generated_at': published.get('generated_at')}
|
||
raise RuntimeError('计划正在切换,请稍后重试')
|