269 lines
13 KiB
Python
269 lines
13 KiB
Python
"""正式ETF观测独立消费者。持久化后确认;不导入交易、信号执行或QMT服务。"""
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import hashlib
|
||
import json
|
||
import re
|
||
from datetime import datetime, timedelta
|
||
from zoneinfo import ZoneInfo
|
||
|
||
from app.core.etf_contract import consumer as contract
|
||
from app.core.etf_intraday_view import intraday_series
|
||
from app.repo import rotation_repo as store
|
||
from app.services import param_store
|
||
from config.settings import settings
|
||
|
||
TZ = ZoneInfo('Asia/Shanghai')
|
||
NS = contract.OBSERVATION_NAMESPACE
|
||
GROUP = 'pms-etf-observation-v1'
|
||
CONSUMER = 'pms-observation'
|
||
PREFIX = 'etf-observation:'
|
||
_clients = {}
|
||
|
||
|
||
def now():
|
||
return datetime.now(TZ)
|
||
|
||
|
||
def day_key(day):
|
||
return PREFIX + 'day:' + day
|
||
|
||
|
||
def initial(day):
|
||
return {'day': day, 'namespace': NS, 'rounds': {}, 'sequences': {}, 'events': {},
|
||
'history': [], 'summary': None, 'last_success_at': None, 'error': None}
|
||
|
||
|
||
def clients():
|
||
import redis
|
||
pair = (settings.PMS_ETF_SNAPSHOT_REDIS_URL, settings.PMS_ETF_EVENT_REDIS_URL)
|
||
if not all(pair):
|
||
raise ValueError('ETF观测连接未配置')
|
||
if pair not in _clients:
|
||
_clients[pair] = tuple(redis.Redis.from_url(url, protocol=2, decode_responses=True,
|
||
socket_connect_timeout=3, socket_timeout=5) for url in pair)
|
||
return _clients[pair]
|
||
|
||
|
||
def bounded(client, key, maximum):
|
||
size = client.strlen(key)
|
||
if type(size) is not int or not 0 < size <= maximum:
|
||
raise ValueError('观测提交数据缺失或超限')
|
||
raw = client.getrange(key, 0, maximum - 1)
|
||
if not isinstance(raw, str) or len(raw.encode()) != size:
|
||
raise ValueError('观测读取期间数据变化')
|
||
return raw
|
||
|
||
|
||
def bundle(snapshots, events, key):
|
||
if not re.fullmatch(re.escape(NS) + r':manifest:\d{4}-\d{2}-\d{2}:\d{8}T\d{4}:r\d+', key):
|
||
raise ValueError('非正式ETF提交键')
|
||
encoded = bounded(events, key, 262144)
|
||
manifest = contract._json(encoded)
|
||
expected = key.replace(':manifest:', ':snapshot:', 1)
|
||
if manifest.get('manifest_key') != key or manifest.get('snapshot_key') != expected:
|
||
raise ValueError('观测提交身份不匹配')
|
||
raw = bounded(snapshots, expected, 16777216)
|
||
if bounded(events, key, 262144) != encoded:
|
||
raise ValueError('观测提交版本变化')
|
||
return manifest, raw
|
||
|
||
|
||
def subject_allowed(kind, code):
|
||
if kind == 'MARKET':
|
||
return code == 'BROAD_ETF'
|
||
if kind == 'INDEX' and code == '932000.CSI':
|
||
return True
|
||
return kind in ('ETF', 'INDEX') and bool(re.fullmatch(r'\d{6}\.(SH|SZ)', str(code)))
|
||
|
||
|
||
def archive_round(manifest, raw):
|
||
key = PREFIX + 'round:' + manifest['trade_date'] + ':' + manifest['round_id'] + ':r' + str(manifest['revision'])
|
||
store.put_once(key, 'etf_observation_round', {'manifest': manifest, 'snapshot_text': raw})
|
||
|
||
|
||
def summary_of(result):
|
||
snap = result['snapshot']
|
||
market = next((s for s in snap['subjects'] if s['subject_type'] == 'MARKET' and s['subject_id'] == 'BROAD_ETF'), {})
|
||
scopes = {p['scope']: p for p in result['scopes'] if p['subject_type'] == 'MARKET' and p['subject_id'] == 'BROAD_ETF'}
|
||
return {key: result.get(key) for key in ('round_id', 'revision', 'published_at', 'valid_until', 'reason_code',
|
||
'observation_eligible', 'model_status', 'calibration_status', 'review_status', 'calibration_review')} | {
|
||
'trade_date': snap['trade_date'], 'window_end': snap['window_end'], 'session_closed': snap['session_closed'],
|
||
'market': market, 'scopes': scopes, 'member_count': market.get('member_count'),
|
||
'trade_instruction': False}
|
||
|
||
|
||
def save_snapshot(manifest, raw, at):
|
||
day = manifest['trade_date']
|
||
with store.locked(day_key(day), 'etf_observation_state', initial(day)) as state:
|
||
revision = state['rounds'].get(manifest['round_id'], -1)
|
||
result = contract.validate_observation_snapshot(manifest, raw, at, last_revision=revision)
|
||
if result['reason_code'] in ('OLD_ROUND_REVISION', 'FUTURE_EVENT', 'REPLAY_NOT_ACTIONABLE'):
|
||
return
|
||
archive_round(manifest, raw)
|
||
state['rounds'][manifest['round_id']] = max(revision, manifest['revision'])
|
||
summary = summary_of(result)
|
||
old = state.get('summary')
|
||
if old is None or (summary['window_end'], summary['revision']) >= (old['window_end'], old['revision']):
|
||
state['summary'] = summary
|
||
history = {s['round_id']: s for s in state.get('history', [])}
|
||
history[summary['round_id']] = summary
|
||
state['history'] = sorted(history.values(), key=lambda s: s['window_end'])[-16:]
|
||
|
||
|
||
def apply_event(stream_id, fields, manifest, raw, at):
|
||
event = contract.decode_stream_entry(fields)
|
||
if not subject_allowed(event.get('subject_type'), event.get('subject_id')):
|
||
raise ValueError('ETF观测主体不在沪深范围')
|
||
audit_key = PREFIX + 'event:' + event['trade_date'] + ':' + stream_id
|
||
# 已提交未ACK时重放,不改变第一次处理结论或实际接收时间。
|
||
if store.get(audit_key)[0] is not None:
|
||
return
|
||
day = event['trade_date']
|
||
with store.locked(day_key(day), 'etf_observation_state', initial(day)) as state:
|
||
if store.get(audit_key)[0] is not None:
|
||
return
|
||
last = state['sequences'].get(event['event_id'], 0)
|
||
decision = contract.validate_envelope(event, manifest, raw, at, last_event_seq=last,
|
||
last_revision=state['rounds'].get(event['round_id'], -1))
|
||
archive_round(manifest, raw)
|
||
store.put_once(audit_key, 'etf_observation_event', {'stream_id': stream_id, 'namespace': NS,
|
||
'raw': fields, 'decision': decision, 'received_at': at.isoformat()})
|
||
if decision['reason_code'] not in ('FUTURE_EVENT', 'OLD_ROUND_REVISION', 'REPLAY_NOT_ACTIONABLE'):
|
||
state['sequences'][event['event_id']] = max(last, event['event_seq'])
|
||
state['rounds'][event['round_id']] = max(state['rounds'].get(event['round_id'], -1), event['revision'])
|
||
if decision['sequence_gap']:
|
||
state['sequence_gap'] = True
|
||
if decision['apply']:
|
||
previous = state['events'].get(event['event_id']) or {}
|
||
subjects = json.loads(raw)['subjects']
|
||
subject = next((s for s in subjects if (s['subject_type'], s['subject_id']) ==
|
||
(event['subject_type'], event['subject_id'])), {})
|
||
state['events'][event['event_id']] = {'event': event, 'decision': decision,
|
||
'received_at': at.isoformat(), 'first_received_at': previous.get('first_received_at', at.isoformat()),
|
||
'observation': subject.get('observation', {}), 'member_count': subject.get('member_count')}
|
||
|
||
|
||
def consume_once(snapshots=None, events=None, clock=now):
|
||
if not param_store.get_bool('PMS_ETF_OBSERVATION_ENABLED', False):
|
||
return {'status': 'disabled'}
|
||
snapshots, events = (snapshots, events) if snapshots is not None else clients()
|
||
day = clock().astimezone(TZ).date().isoformat()
|
||
stream = NS + ':events:' + day
|
||
processed = 0
|
||
failures = []
|
||
try:
|
||
try:
|
||
events.xgroup_create(stream, GROUP, id='0-0', mkstream=True)
|
||
except Exception as exc:
|
||
if 'BUSYGROUP' not in str(exc):
|
||
raise
|
||
# 固定消费者名,重启先读取自己的未确认项;不触碰别的消费组。
|
||
for start in ('0-0', '>'):
|
||
for _key, rows in events.xreadgroup(GROUP, CONSUMER, {stream: start}, count=100):
|
||
for stream_id, fields in rows:
|
||
try:
|
||
event = contract.decode_stream_entry(fields)
|
||
if event.get('trade_date') != day:
|
||
raise ValueError('事件日期与流日期不一致')
|
||
key = f"{NS}:manifest:{day}:{event['round_id']}:r{event['revision']}"
|
||
manifest, raw = bundle(snapshots, events, key)
|
||
apply_event(stream_id, fields, manifest, raw, clock())
|
||
# put_once、状态和水位已经持久化;数据库故障时不能走到这里。
|
||
events.xack(stream, GROUP, stream_id)
|
||
processed += 1
|
||
except Exception as exc:
|
||
failures.append(type(exc).__name__)
|
||
pointer = snapshots.get(NS + ':latest')
|
||
keys = []
|
||
if pointer:
|
||
manifest = contract._json(pointer)
|
||
if manifest.get('trade_date') == day:
|
||
keys.append(manifest['manifest_key'])
|
||
# 收盘快照不推进活动latest,显式检查当日固定收盘轮及其最高可见修订。
|
||
if clock().astimezone(TZ).hour >= 15:
|
||
close_keys = list(events.scan_iter(match=f'{NS}:manifest:{day}:{day.replace("-", "")}T1500:r*', count=20))
|
||
if close_keys:
|
||
keys.append(max(close_keys, key=lambda k: int(k.rsplit(':r', 1)[1])))
|
||
for key in dict.fromkeys(keys):
|
||
manifest, raw = bundle(snapshots, events, key)
|
||
save_snapshot(manifest, raw, clock())
|
||
with store.locked(day_key(day), 'etf_observation_state', initial(day)) as state:
|
||
if failures:
|
||
state.update(error=failures[0], last_failure_at=clock().isoformat())
|
||
else:
|
||
state.update(last_success_at=clock().isoformat(), error=None)
|
||
return {'status': 'unavailable' if failures else 'ok', 'error': failures[0] if failures else None, 'processed': processed}
|
||
except Exception as exc:
|
||
# 不输出连接串、上游原文或密钥。坏包保留pending,下一轮补齐后恢复。
|
||
with store.locked(day_key(day), 'etf_observation_state', initial(day)) as state:
|
||
state['error'] = type(exc).__name__
|
||
state['last_failure_at'] = clock().isoformat()
|
||
return {'status': 'unavailable', 'error': type(exc).__name__, 'processed': processed}
|
||
|
||
|
||
def name_map():
|
||
"""只读ETF基础资料;缺名显示代码,不借用同代码股票名称。"""
|
||
from app.db import session as db
|
||
try:
|
||
rows = db.fetch_all('SELECT ts_code,csname,index_code,index_name FROM gp_etf_basic WHERE ts_code LIKE :sh OR ts_code LIKE :sz', {'sh': '%.SH', 'sz': '%.SZ'})
|
||
result = {}
|
||
for row in rows:
|
||
code = str(row.get('ts_code') or '')
|
||
if subject_allowed('ETF', code):
|
||
result[code] = str(row.get('csname') or code)
|
||
index = str(row.get('index_code') or '')
|
||
if subject_allowed('INDEX', index):
|
||
result[index] = str(row.get('index_name') or index)
|
||
return result
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def short_name(name, code):
|
||
text = str(name or code)
|
||
for old, new in [('工银瑞信', '工银'), ('华泰柏瑞', '华泰'), ('上证科创板50成份增强策略', '科创50'),
|
||
('上证科创板50成份', '科创50'), ('中证A500', 'A500'), ('中证2000', '2000'), ('交易型开放式指数证券投资基金', ''),
|
||
('交易型开放式指数', ''), ('证券投资基金', ''), ('ETF', '')]:
|
||
text = text.replace(old, new)
|
||
return text if 0 < sum(1 if ord(c) < 128 else 2 for c in text) <= 10 else code.split('.')[0]
|
||
|
||
|
||
def presentation(state, at, names=None):
|
||
names = names or {}
|
||
data = copy.deepcopy(state)
|
||
summary = data.get('summary')
|
||
healthy = bool(data.get('last_success_at')) and (at - datetime.fromisoformat(data['last_success_at'])).total_seconds() <= 90 and not data.get('error')
|
||
live = bool(summary and any(p.get('observation_eligible') for p in summary.get('scopes', {}).values()) and datetime.fromisoformat(summary['valid_until']) > at)
|
||
data['status'] = ('unavailable' if not healthy else 'waiting' if not summary else
|
||
'closed' if summary.get('session_closed') else 'live' if live else 'expired')
|
||
rows = []
|
||
for item in data.get('events', {}).values():
|
||
e = item['event']
|
||
full = names.get(e['subject_id']) or ('宽基ETF' if e['subject_type'] == 'MARKET' else e['subject_id'])
|
||
stale = e['event_state'] not in contract.ACTIVE_STATES or at >= datetime.fromisoformat(e['valid_until'])
|
||
rows.append({**e, 'name': full if e['subject_type'] == 'INDEX' and len(full) <= 9 else short_name(full, e['subject_id']), 'full_name': full,
|
||
'observation': item['observation'], 'member_count': item.get('member_count'),
|
||
'received_at': item['received_at'], 'stale': stale,
|
||
'display_state': 'EXPIRED' if stale and e['event_state'] in contract.ACTIVE_STATES else e['event_state']})
|
||
data.update(events=sorted(rows, key=lambda e: (e['published_at'], e['event_id']), reverse=True),
|
||
enabled=True, trade_instruction=False, server_at=at.isoformat(), current_eligible=live)
|
||
data.pop('sequences', None)
|
||
data.pop('rounds', None)
|
||
data['intraday_series'] = intraday_series(data.get('history'), data.get('day'), at)
|
||
return data
|
||
|
||
|
||
_view_names = {'at': None, 'data': {}}
|
||
|
||
|
||
def status():
|
||
if not param_store.get_bool('PMS_ETF_OBSERVATION_ENABLED', False):
|
||
return {'enabled': False, 'status': 'disabled', 'events': [], 'trade_instruction': False}
|
||
at = now()
|
||
state = store.get(day_key(at.date().isoformat()))[0] or initial(at.date().isoformat())
|
||
if _view_names['at'] is None or (at - _view_names['at']).total_seconds() >= 3600:
|
||
_view_names.update(at=at, data=name_map())
|
||
return presentation(state, at, _view_names['data'])
|