151 lines
7.7 KiB
Python
151 lines
7.7 KiB
Python
"""宽基每日资金只读接入;原文留存后确认,仅供页面观测。"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import re
|
||
from datetime import datetime, timedelta
|
||
|
||
from app.core.etf_contract import daily_transport as contract
|
||
from app.repo import rotation_repo as store
|
||
from app.services import etf_observation as minute, param_store
|
||
|
||
NS = contract.NAMESPACE
|
||
GROUP = 'pms-broad-daily-v1'
|
||
STATE = 'broad-funds:state'
|
||
INDEX_NAMES = {'000016.SH': '上证50', '000300.SH': '沪深300', '000510.SH': '中证A500',
|
||
'000688.SH': '科创50', '000852.SH': '中证1000', '000905.SH': '中证500',
|
||
'399006.SZ': '创业板指', '932000.CSI': '中证2000'}
|
||
|
||
|
||
def initial():
|
||
return {'days': {}, 'latest': None, 'last_success_at': None, 'error': None}
|
||
|
||
|
||
def validate_scope(report, at):
|
||
if report['trade_date'] > at.date().isoformat() or datetime.fromisoformat(report['published_at']) > at:
|
||
raise ValueError('DAILY_FUTURE_FACT')
|
||
for code in report['etf'].get('cohort', []):
|
||
if not re.fullmatch(r'\d{6}\.(SH|SZ)', str(code)):
|
||
raise ValueError('DAILY_OUT_OF_SCOPE')
|
||
for row in report['etf'].get('etfs', []):
|
||
if not minute.subject_allowed('ETF', row['subject_id']):
|
||
raise ValueError('DAILY_OUT_OF_SCOPE')
|
||
for row in report['etf'].get('indexes', []):
|
||
if not minute.subject_allowed('INDEX', row['subject_id']):
|
||
raise ValueError('DAILY_OUT_OF_SCOPE')
|
||
|
||
|
||
def ingest(snapshots, events, manifest_key, day, at, delivery=None):
|
||
"""契约核验、同日修订防倒退、事实与回执在同一事务落库。"""
|
||
audit_key = 'broad-funds:event:' + delivery[0] + ':' + delivery[1] if delivery else None
|
||
if audit_key and store.get(audit_key)[0] is not None:
|
||
return
|
||
result = contract.read_daily_report(snapshots, events, manifest_key, expected_trade_date=day)
|
||
report, manifest = result['report'], result['manifest']
|
||
validate_scope(report, at)
|
||
if manifest.get('event_type') != 'DAILY_UPSERT':
|
||
raise ValueError('DAILY_EVENT_TYPE_MISMATCH')
|
||
raw = contract._read(snapshots, manifest['snapshot_key'], contract.MAX_REPORT_BYTES)
|
||
if hashlib.sha256(raw).hexdigest() != manifest['snapshot_sha256']:
|
||
raise ValueError('DAILY_ARCHIVE_HASH_MISMATCH')
|
||
if delivery:
|
||
fields = delivery[2]
|
||
event = contract._json(fields.get('data'))
|
||
expected = dict(manifest, event_id=f"daily-flow:{day}:r{report['revision']}",
|
||
supersedes_revision=report['supersedes_revision'], component_status=report['component_status'])
|
||
if event != expected:
|
||
raise ValueError('DAILY_EVENT_COMMIT_MISMATCH')
|
||
key = f"broad-funds:report:{day}:r{report['revision']}"
|
||
with store.locked(STATE, 'broad_funds_state', initial()) as state:
|
||
if audit_key and store.get(audit_key)[0] is not None:
|
||
return
|
||
store.put_once(key, 'broad_funds_report', {'manifest': manifest, 'snapshot_text': raw.decode('utf-8')})
|
||
old = state['days'].get(day, {})
|
||
apply = not result['superseded'] and report['revision'] > old.get('revision', -1)
|
||
if apply:
|
||
state['days'][day] = {'revision': report['revision'], 'key': key,
|
||
'component_status': report['component_status'], 'published_at': report['published_at']}
|
||
if not state['latest'] or day >= state['latest']:
|
||
state['latest'] = day
|
||
if audit_key:
|
||
store.put_once(audit_key, 'broad_funds_event', {'stream': delivery[0], 'stream_id': delivery[1],
|
||
'raw': delivery[2], 'manifest_key': manifest_key, 'applied': apply,
|
||
'superseded': result['superseded'], 'received_at': at.isoformat()})
|
||
# 只限制状态索引大小,原始快照与事件审计均保留。
|
||
state['days'] = {k: state['days'][k] for k in sorted(state['days'])[-100:]}
|
||
|
||
|
||
def consume_once(snapshots=None, events=None, clock=minute.now):
|
||
if not param_store.get_bool('PMS_BROAD_DAILY_ENABLED', False):
|
||
return {'status': 'disabled'}
|
||
snapshots, events = (snapshots, events) if snapshots is not None else minute.clients()
|
||
at = clock().astimezone(minute.TZ)
|
||
failures, processed = [], 0
|
||
# 正式保留期内的月流,固定边界,不扫描整个Redis,也不触碰交易消费组。
|
||
months = sorted({(at.date() - timedelta(days=n)).isoformat()[:7] for n in range(91)})
|
||
try:
|
||
for month in months:
|
||
stream = NS + ':events:' + month
|
||
if not events.exists(stream):
|
||
continue
|
||
try:
|
||
events.xgroup_create(stream, GROUP, id='0-0')
|
||
except Exception as exc:
|
||
if 'BUSYGROUP' not in str(exc):
|
||
raise
|
||
for cursor in ('0-0', '>'):
|
||
for _stream, rows in events.xreadgroup(GROUP, minute.CONSUMER, {stream: cursor}, count=100):
|
||
for sid, fields in rows:
|
||
try:
|
||
event = contract._json(fields.get('data'))
|
||
day = event['trade_date']
|
||
if day[:7] != month:
|
||
raise ValueError('DAILY_STREAM_DATE_MISMATCH')
|
||
ingest(snapshots, events, event['manifest_key'], day, clock(), (stream, sid, fields))
|
||
events.xack(stream, GROUP, sid)
|
||
processed += 1
|
||
except Exception as exc:
|
||
failures.append(type(exc).__name__)
|
||
# 即使月流暂缺或某条待补,也核验最新正式提交;没有事实时不沿用旧日冒充今天。
|
||
if snapshots.strlen(NS + ':latest'):
|
||
pointer = contract._json(contract._read(snapshots, NS + ':latest', 32768))
|
||
manifest = contract._json(contract._read(events, pointer['manifest_key'], 32768))
|
||
if pointer != manifest:
|
||
raise ValueError('DAILY_POINTER_NOT_COMMITTED')
|
||
ingest(snapshots, events, pointer['manifest_key'], pointer['trade_date'], clock())
|
||
except Exception as exc:
|
||
failures.append(type(exc).__name__)
|
||
with store.locked(STATE, 'broad_funds_state', initial()) as state:
|
||
if failures:
|
||
state.update(error=failures[0], last_failure_at=clock().isoformat())
|
||
else:
|
||
state.update(error=None, last_success_at=clock().isoformat())
|
||
return {'status': 'unavailable' if failures else 'ok', 'processed': processed,
|
||
'error': failures[0] if failures else None}
|
||
|
||
|
||
def status(clock=minute.now):
|
||
if not param_store.get_bool('PMS_BROAD_DAILY_ENABLED', False):
|
||
return {'enabled': False, 'trade_instruction': False}
|
||
state = store.get(STATE)[0] or initial()
|
||
entry = state['days'].get(state['latest'], {})
|
||
saved = store.get(entry['key'])[0] if entry.get('key') else None
|
||
at = clock()
|
||
healthy = bool(state['last_success_at']) and not state['error'] and (
|
||
at - datetime.fromisoformat(state['last_success_at'])).total_seconds() <= 180
|
||
result = {'enabled': True, 'status': 'ok' if healthy and saved else 'waiting' if healthy else 'unavailable',
|
||
'trade_instruction': False, 'last_success_at': state['last_success_at'], 'error': state['error'],
|
||
'server_at': at.isoformat(), 'report': None}
|
||
if saved:
|
||
report = json.loads(saved['snapshot_text'])
|
||
# 页面只需市场、指数、行业;完整逐基金事实保存在不可变档案中。
|
||
for key in ('etfs', 'cohort'):
|
||
report['etf'].pop(key, None)
|
||
report.pop('source_observations', None)
|
||
for row in report['etf'].get('indexes', []):
|
||
row['name'] = INDEX_NAMES.get(row['subject_id']) or row.get('name') or row['subject_id']
|
||
result['report'] = report
|
||
result['manifest'] = saved['manifest']
|
||
return result
|