tradingSystem/app/services/etf_observation.py

272 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""正式ETF观测独立消费者。持久化后确认;不导入交易、信号执行或QMT服务。"""
from __future__ import annotations
import copy
import hashlib
import json
import re
from datetime import datetime, timedelta
from zoneinfo import ZoneInfo
from app.core.etf_contract import consumer as contract
from app.core.etf_intraday_view import intraday_series
from app.repo import rotation_repo as store
from app.services import param_store
from config.settings import settings
TZ = ZoneInfo('Asia/Shanghai')
NS = contract.OBSERVATION_NAMESPACE
GROUP = 'pms-etf-observation-v1'
CONSUMER = 'pms-observation'
PREFIX = 'etf-observation:'
_clients = {}
def now():
return datetime.now(TZ)
def day_key(day):
return PREFIX + 'day:' + day
def initial(day):
return {'day': day, 'namespace': NS, 'rounds': {}, 'sequences': {}, 'events': {},
'history': [], 'summary': None, 'last_success_at': None, 'error': None}
def clients():
import redis
pair = (settings.PMS_ETF_SNAPSHOT_REDIS_URL, settings.PMS_ETF_EVENT_REDIS_URL)
if not all(pair):
raise ValueError('ETF观测连接未配置')
if pair not in _clients:
_clients[pair] = tuple(redis.Redis.from_url(url, protocol=2, decode_responses=True,
socket_connect_timeout=3, socket_timeout=5) for url in pair)
return _clients[pair]
def bounded(client, key, maximum):
size = client.strlen(key)
if type(size) is not int or not 0 < size <= maximum:
raise ValueError('观测提交数据缺失或超限')
raw = client.getrange(key, 0, maximum - 1)
if not isinstance(raw, str) or len(raw.encode()) != size:
raise ValueError('观测读取期间数据变化')
return raw
def bundle(snapshots, events, key):
if not re.fullmatch(re.escape(NS) + r':manifest:\d{4}-\d{2}-\d{2}:\d{8}T\d{4}:r\d+', key):
raise ValueError('非正式ETF提交键')
encoded = bounded(events, key, 262144)
manifest = contract._json(encoded)
expected = key.replace(':manifest:', ':snapshot:', 1)
if manifest.get('manifest_key') != key or manifest.get('snapshot_key') != expected:
raise ValueError('观测提交身份不匹配')
raw = bounded(snapshots, expected, 16777216)
if bounded(events, key, 262144) != encoded:
raise ValueError('观测提交版本变化')
return manifest, raw
def subject_allowed(kind, code):
if kind == 'MARKET':
return code == 'BROAD_ETF'
if kind == 'INDEX' and code == '932000.CSI':
return True
return kind in ('ETF', 'INDEX') and bool(re.fullmatch(r'\d{6}\.(SH|SZ)', str(code)))
def archive_round(manifest, raw):
key = PREFIX + 'round:' + manifest['trade_date'] + ':' + manifest['round_id'] + ':r' + str(manifest['revision'])
store.put_once(key, 'etf_observation_round', {'manifest': manifest, 'snapshot_text': raw})
def summary_of(result):
snap = result['snapshot']
market = next((s for s in snap['subjects'] if s['subject_type'] == 'MARKET' and s['subject_id'] == 'BROAD_ETF'), {})
scopes = {p['scope']: p for p in result['scopes'] if p['subject_type'] == 'MARKET' and p['subject_id'] == 'BROAD_ETF'}
return {key: result.get(key) for key in ('round_id', 'revision', 'published_at', 'valid_until', 'reason_code',
'observation_eligible', 'model_status', 'calibration_status', 'review_status', 'calibration_review')} | {
'trade_date': snap['trade_date'], 'window_end': snap['window_end'], 'session_closed': snap['session_closed'],
'market': market, 'scopes': scopes, 'member_count': market.get('member_count'),
'trade_instruction': False}
def save_snapshot(manifest, raw, at):
day = manifest['trade_date']
with store.locked(day_key(day), 'etf_observation_state', initial(day)) as state:
revision = state['rounds'].get(manifest['round_id'], -1)
result = contract.validate_observation_snapshot(manifest, raw, at, last_revision=revision)
if result['reason_code'] in ('OLD_ROUND_REVISION', 'FUTURE_EVENT', 'REPLAY_NOT_ACTIONABLE'):
return
archive_round(manifest, raw)
state['rounds'][manifest['round_id']] = max(revision, manifest['revision'])
summary = summary_of(result)
old = state.get('summary')
if old is None or (summary['window_end'], summary['revision']) >= (old['window_end'], old['revision']):
state['summary'] = summary
history = {s['round_id']: s for s in state.get('history', [])}
history[summary['round_id']] = summary
state['history'] = sorted(history.values(), key=lambda s: s['window_end'])[-16:]
def apply_event(stream_id, fields, manifest, raw, at):
event = contract.decode_stream_entry(fields)
if not subject_allowed(event.get('subject_type'), event.get('subject_id')):
raise ValueError('ETF观测主体不在沪深范围')
audit_key = PREFIX + 'event:' + event['trade_date'] + ':' + stream_id
# 已提交未ACK时重放,不改变第一次处理结论或实际接收时间。
if store.get(audit_key)[0] is not None:
return
day = event['trade_date']
with store.locked(day_key(day), 'etf_observation_state', initial(day)) as state:
if store.get(audit_key)[0] is not None:
return
last = state['sequences'].get(event['event_id'], 0)
decision = contract.validate_envelope(event, manifest, raw, at, last_event_seq=last,
last_revision=state['rounds'].get(event['round_id'], -1))
archive_round(manifest, raw)
store.put_once(audit_key, 'etf_observation_event', {'stream_id': stream_id, 'namespace': NS,
'raw': fields, 'decision': decision, 'received_at': at.isoformat()})
if decision['reason_code'] not in ('FUTURE_EVENT', 'OLD_ROUND_REVISION', 'REPLAY_NOT_ACTIONABLE'):
state['sequences'][event['event_id']] = max(last, event['event_seq'])
state['rounds'][event['round_id']] = max(state['rounds'].get(event['round_id'], -1), event['revision'])
if decision['sequence_gap']:
state['sequence_gap'] = True
if decision['apply']:
previous = state['events'].get(event['event_id']) or {}
subjects = json.loads(raw)['subjects']
subject = next((s for s in subjects if (s['subject_type'], s['subject_id']) ==
(event['subject_type'], event['subject_id'])), {})
state['events'][event['event_id']] = {'event': event, 'decision': decision,
'received_at': at.isoformat(), 'first_received_at': previous.get('first_received_at', at.isoformat()),
'observation': subject.get('observation', {}), 'member_count': subject.get('member_count')}
def consume_once(snapshots=None, events=None, clock=now):
if not param_store.get_bool('PMS_ETF_OBSERVATION_ENABLED', False):
return {'status': 'disabled'}
snapshots, events = (snapshots, events) if snapshots is not None else clients()
day = clock().astimezone(TZ).date().isoformat()
stream = NS + ':events:' + day
processed = 0
failures = []
try:
try:
events.xgroup_create(stream, GROUP, id='0-0', mkstream=True)
except Exception as exc:
if 'BUSYGROUP' not in str(exc):
raise
# 固定消费者名,重启先读取自己的未确认项;不触碰别的消费组。
for start in ('0-0', '>'):
for _key, rows in events.xreadgroup(GROUP, CONSUMER, {stream: start}, count=100):
for stream_id, fields in rows:
try:
event = contract.decode_stream_entry(fields)
if event.get('trade_date') != day:
raise ValueError('事件日期与流日期不一致')
key = f"{NS}:manifest:{day}:{event['round_id']}:r{event['revision']}"
manifest, raw = bundle(snapshots, events, key)
apply_event(stream_id, fields, manifest, raw, clock())
# put_once、状态和水位已经持久化;数据库故障时不能走到这里。
events.xack(stream, GROUP, stream_id)
processed += 1
except Exception as exc:
failures.append(type(exc).__name__)
pointer = snapshots.get(NS + ':latest')
keys = []
if pointer:
manifest = contract._json(pointer)
if manifest.get('trade_date') == day:
keys.append(manifest['manifest_key'])
# 收盘快照不推进活动latest,显式检查当日固定收盘轮及其最高可见修订。
if clock().astimezone(TZ).hour >= 15:
close_keys = list(events.scan_iter(match=f'{NS}:manifest:{day}:{day.replace("-", "")}T1500:r*', count=20))
if close_keys:
keys.append(max(close_keys, key=lambda k: int(k.rsplit(':r', 1)[1])))
for key in dict.fromkeys(keys):
manifest, raw = bundle(snapshots, events, key)
save_snapshot(manifest, raw, clock())
with store.locked(day_key(day), 'etf_observation_state', initial(day)) as state:
if failures:
state.update(error=failures[0], last_failure_at=clock().isoformat())
else:
state.update(last_success_at=clock().isoformat(), error=None)
return {'status': 'unavailable' if failures else 'ok', 'error': failures[0] if failures else None, 'processed': processed}
except Exception as exc:
# 不输出连接串、上游原文或密钥。坏包保留pending,下一轮补齐后恢复。
with store.locked(day_key(day), 'etf_observation_state', initial(day)) as state:
state['error'] = type(exc).__name__
state['last_failure_at'] = clock().isoformat()
return {'status': 'unavailable', 'error': type(exc).__name__, 'processed': processed}
INDEX_LABELS = {
'000016.SH': '上证50', '000300.SH': '沪深300',
'000905.SH': '中证500', '000852.SH': '中证1000',
'932000.CSI': '中证2000', '000510.SH': '中证A500',
'000688.SH': '科创50', '399006.SZ': '创业板指',
}
def name_map():
"""只读基金全名与跟踪指数;简称按指数代码确定,不从基金名称猜测。"""
from app.db import session as db
try:
rows = db.fetch_all('SELECT ts_code,csname,index_code,index_name FROM gp_etf_basic WHERE ts_code LIKE :sh OR ts_code LIKE :sz', {'sh': '%.SH', 'sz': '%.SZ'})
result = {}
for row in rows:
code = str(row.get('ts_code') or '')
if not subject_allowed('ETF', code):
continue
index = str(row.get('index_code') or '')
result[code] = {'full_name': str(row.get('csname') or code), 'index_code': index}
if subject_allowed('INDEX', index):
result[index] = {'full_name': str(row.get('index_name') or index), 'index_code': index}
return result
except Exception:
return {}
def presentation(state, at, names=None):
names = names or {}
data = copy.deepcopy(state)
summary = data.get('summary')
healthy = bool(data.get('last_success_at')) and (at - datetime.fromisoformat(data['last_success_at'])).total_seconds() <= 90 and not data.get('error')
live = bool(summary and any(p.get('observation_eligible') for p in summary.get('scopes', {}).values()) and datetime.fromisoformat(summary['valid_until']) > at)
data['status'] = ('unavailable' if not healthy else 'waiting' if not summary else
'closed' if summary.get('session_closed') else 'live' if live else 'expired')
rows = []
for item in data.get('events', {}).values():
e = item['event']
profile = names.get(e['subject_id']) or {}
full = profile.get('full_name') or ('宽基ETF' if e['subject_type'] == 'MARKET' else e['subject_id'])
index = e['subject_id'] if e['subject_type'] == 'INDEX' else profile.get('index_code')
label = '宽基' if e['subject_type'] == 'MARKET' else INDEX_LABELS.get(index, e['subject_id'].split('.')[0])
stale = e['event_state'] not in contract.ACTIVE_STATES or at >= datetime.fromisoformat(e['valid_until'])
rows.append({**e, 'name': label, 'full_name': full,
'observation': item['observation'], 'member_count': item.get('member_count'),
'received_at': item['received_at'], 'stale': stale,
'display_state': 'EXPIRED' if stale and e['event_state'] in contract.ACTIVE_STATES else e['event_state']})
data.update(events=sorted(rows, key=lambda e: (e['published_at'], e['event_id']), reverse=True),
enabled=True, trade_instruction=False, server_at=at.isoformat(), current_eligible=live)
data.pop('sequences', None)
data.pop('rounds', None)
data['intraday_series'] = intraday_series(data.get('history'), data.get('day'), at)
return data
_view_names = {'at': None, 'data': {}}
def status():
if not param_store.get_bool('PMS_ETF_OBSERVATION_ENABLED', False):
return {'enabled': False, 'status': 'disabled', 'events': [], 'trade_instruction': False}
at = now()
state = store.get(day_key(at.date().isoformat()))[0] or initial(at.date().isoformat())
if _view_names['at'] is None or (at - _view_names['at']).total_seconds() >= 3600:
_view_names.update(at=at, data=name_map())
return presentation(state, at, _view_names['data'])