212 lines
8.8 KiB
Python
212 lines
8.8 KiB
Python
"""只追加的计划版本。目录内文件锁涵盖采集、校验与发布,最新文件原子替换。"""
|
||
from __future__ import annotations
|
||
|
||
import contextlib
|
||
import copy
|
||
import datetime as dt
|
||
import fcntl
|
||
import hashlib
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import tempfile
|
||
|
||
|
||
def digest(value):
|
||
return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True,
|
||
separators=(',', ':'), allow_nan=False).encode()).hexdigest()
|
||
|
||
|
||
def now():
|
||
return dt.datetime.now(dt.timezone.utc).isoformat(timespec='seconds')
|
||
|
||
|
||
def _date(day):
|
||
return dt.date.fromisoformat(day).isoformat()
|
||
|
||
|
||
def atomic(path, value, *, text=False):
|
||
path = Path(path)
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
fd, tmp = tempfile.mkstemp(prefix='.' + path.name, dir=path.parent)
|
||
try:
|
||
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
||
if text:
|
||
f.write(value)
|
||
else:
|
||
json.dump(value, f, ensure_ascii=False, sort_keys=True, allow_nan=False)
|
||
f.flush()
|
||
os.fsync(f.fileno())
|
||
os.replace(tmp, path)
|
||
directory = os.open(str(path.parent), os.O_RDONLY)
|
||
try:
|
||
os.fsync(directory)
|
||
finally:
|
||
os.close(directory)
|
||
finally:
|
||
if os.path.exists(tmp):
|
||
os.unlink(tmp)
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def writer(root):
|
||
Path(root).mkdir(parents=True, exist_ok=True)
|
||
with open(Path(root) / '.publish.lock', 'a') as lock:
|
||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||
yield
|
||
|
||
|
||
def latest_date(root):
|
||
days = sorted(p.stem[5:] for p in Path(root).glob('plan_????-??-??.json'))
|
||
return days[-1] if days else None
|
||
|
||
|
||
def load(root, day=None, decision_id=None, morning=False):
|
||
if decision_id:
|
||
if not re.fullmatch(r'\d{4}-\d{2}-\d{2}_[0-9a-f]{64}', decision_id):
|
||
raise ValueError('计划版本标识无效')
|
||
if day and day != decision_id[:10]:
|
||
raise ValueError('计划日期与版本不匹配')
|
||
latest = load(root, decision_id[:10])
|
||
d = latest.get('decision') or {}
|
||
if 'ancestor_ids' not in d:
|
||
for published in history(root, decision_id[:10]):
|
||
if published['decision']['id'] == decision_id:
|
||
return published
|
||
raise FileNotFoundError('该版本尚未发布')
|
||
if decision_id != d.get('id') and decision_id not in d['ancestor_ids']:
|
||
raise FileNotFoundError('该版本尚未发布')
|
||
path = Path(root) / 'versions' / decision_id[:10] / (decision_id + '.json')
|
||
result = json.loads(path.read_text())['plan']
|
||
if result.get('decision', {}).get('id') != decision_id:
|
||
raise ValueError('归档版本身份不一致')
|
||
if decision_id == d.get('id') and result != latest:
|
||
raise ValueError('发布入口与不可变版本不一致')
|
||
return result
|
||
day = _date(day or latest_date(root) or '')
|
||
if morning:
|
||
pointer = Path(root) / 'versions' / day / 'morning.json'
|
||
if not pointer.exists():
|
||
# 最新指针切换成功、晨版指针写入前中断:沿已发布父链恢复首版。
|
||
versions = history(root, day)
|
||
if not versions:
|
||
raise FileNotFoundError('无可核验晨版')
|
||
return versions[0]
|
||
return load(root, day, json.loads(pointer.read_text())['id'])
|
||
return json.loads((Path(root) / f'plan_{day}.json').read_text())
|
||
|
||
|
||
def envelope(root, decision_id):
|
||
load(root, decision_id=decision_id) # 校验标识,不接受任意文件路径
|
||
return json.loads((Path(root) / 'versions' / decision_id[:10] / (decision_id + '.json')).read_text())
|
||
|
||
|
||
def history(root, day):
|
||
"""只沿已发布父链,目录里未切换成功的文件不是发布历史。"""
|
||
current = load(root, day)
|
||
result, seen = [], set()
|
||
while current.get('decision'):
|
||
d = current['decision']
|
||
if not re.fullmatch(re.escape(day) + r'_[0-9a-f]{64}', d['id']):
|
||
raise ValueError('计划版本标识无效')
|
||
archived = json.loads((Path(root) / 'versions' / day / (d['id'] + '.json')).read_text())['plan']
|
||
if archived != current:
|
||
raise ValueError('发布入口与不可变版本不一致')
|
||
if d['id'] in seen:
|
||
raise ValueError('计划版本父链成环')
|
||
seen.add(d['id'])
|
||
result.append(archived)
|
||
if not d.get('parent_id'):
|
||
break
|
||
parent = d['parent_id']
|
||
if not re.fullmatch(re.escape(day) + r'_[0-9a-f]{64}', parent):
|
||
raise ValueError('计划父版本标识无效')
|
||
current = json.loads((Path(root) / 'versions' / day / (parent + '.json')).read_text())['plan']
|
||
return list(reversed(result))
|
||
|
||
|
||
def health(root, day, **values):
|
||
path = Path(root) / 'versions' / _date(day) / 'health.json'
|
||
if values:
|
||
atomic(path, {'checked_at': now(), **values})
|
||
return json.loads(path.read_text()) if path.exists() else {}
|
||
|
||
|
||
def research(cr, previous=None):
|
||
cr = cr or {}
|
||
previous = previous or {}
|
||
keys = ('overall', 'valuation', 'doubt_hard', 'expectation', 'cycle', 'period_fi',
|
||
'period_rf', 'n_materials', 'unknown_n', 'numbers_unmatched', 'thesis', 'antithesis')
|
||
return {k: copy.deepcopy(cr.get(k)) for k in keys} | {
|
||
'confidence': cr.get('confidence'), 'review_ref': cr.get('review_ref'),
|
||
'changes': [{'field': k, 'before': previous.get(k), 'after': cr.get(k)}
|
||
for k in keys if previous and previous.get(k) != cr.get(k)],
|
||
'checks': [{'kind': k, 'text': cr.get(k), 'status': '待核验'}
|
||
for k in ('catalyst', 'invalidation') if cr.get(k)],
|
||
'report_ran_at': cr.get('report_ran_at') or cr.get('ran_at'),
|
||
'material_status': cr.get('qual'),
|
||
'report_version_url': cr.get('report_version_url'),
|
||
}
|
||
|
||
|
||
def publish(root, plan, inputs, params, *, cutoff_at, reason):
|
||
"""调用方必须持 writer。指纹包含代码、固定输入及配置;时间不导致空版本。"""
|
||
day = _date(plan['date'])
|
||
old = None
|
||
try:
|
||
old = load(root, day)
|
||
except FileNotFoundError:
|
||
pass
|
||
ih, ph = digest(inputs), digest(params)
|
||
prior = (old or {}).get('decision') or {}
|
||
if (prior.get('input_hash') == ih and prior.get('params_hash') == ph
|
||
and old.get('plan_version') == plan.get('plan_version')):
|
||
health(root, day, pending_codes=[], error=None, decision_id=prior['id'])
|
||
return old
|
||
identity = digest({'input': ih, 'params': ph, 'code': plan.get('plan_version'),
|
||
'date': day, 'parent': prior.get('id')})
|
||
did = day + '_' + identity
|
||
if (old or {}).get('decision', {}).get('id') == did:
|
||
return old
|
||
p = copy.deepcopy(plan)
|
||
prev = {r['code']: r for bucket in ('main', 'observe') for r in (old or {}).get(bucket, [])}
|
||
by_code = {}
|
||
rank = 0
|
||
for bucket in ('main', 'observe'):
|
||
for row in p.get(bucket, []):
|
||
rank += 1
|
||
if row['code'] in by_code:
|
||
raise ValueError('完整计划出现重复股票: ' + row['code'])
|
||
row['research_rank'] = rank
|
||
row['review_ref'] = (row.get('company_review') or {}).get('review_ref')
|
||
row['research'] = research(row.get('company_review'), (prev.get(row['code']) or {}).get('company_review'))
|
||
by_code[row['code']] = row
|
||
for bucket in ('candidates', 'watch', 'main_shown', 'observe_shown'):
|
||
p[bucket] = [{**copy.deepcopy(by_code[r['code']]), 'rank': r.get('rank', i)}
|
||
for i, r in enumerate(p.get(bucket, []), 1)]
|
||
p['schema_version'] = 2
|
||
ancestors = prior.get('ancestor_ids')
|
||
if prior and ancestors is None:
|
||
ancestors = [v['decision']['id'] for v in history(root, day)[:-1]]
|
||
p['decision'] = {'id': did, 'parent_id': (old or {}).get('decision', {}).get('id'),
|
||
'ancestor_ids': ((ancestors or []) + [prior['id']]) if prior else [],
|
||
'cutoff_at': cutoff_at, 'input_hash': ih, 'params_hash': ph, 'reason': reason}
|
||
p['generated_at'] = now()
|
||
p['traceability'] = 'captured_inputs'
|
||
# 返回与落盘同一 JSON 形状(元组等不得在内存出口与历史出口有两种表示)。
|
||
p = json.loads(json.dumps(p, ensure_ascii=False, allow_nan=False))
|
||
version_dir = Path(root) / 'versions' / day
|
||
path = version_dir / (did + '.json')
|
||
if path.exists():
|
||
# 中断重试恢复已经落盘但尚未切换的同一版本。
|
||
p = json.loads(path.read_text())['plan']
|
||
else:
|
||
atomic(path, {'plan': p, 'inputs': inputs, 'params': params})
|
||
morning = version_dir / 'morning.json'
|
||
atomic(Path(root) / f'plan_{day}.json', p)
|
||
if not morning.exists():
|
||
atomic(morning, {'id': history(root, day)[0]['decision']['id']})
|
||
health(root, day, pending_codes=[], error=None, decision_id=did)
|
||
return p
|