251 lines
13 KiB
Python
251 lines
13 KiB
Python
"""155模拟实例的独立限制价采集;不调用交易、研判或QMT动作。"""
|
||
import copy
|
||
import re
|
||
import time
|
||
import uuid
|
||
from datetime import datetime, timedelta
|
||
|
||
from config.settings import settings
|
||
from app.core import price_limits as core, tradedays
|
||
from app.core.execution_policy import local_now, timestamp
|
||
from app.db import session as db
|
||
from app.repo import rotation_repo as store
|
||
from app.services import param_store, price_limit_targets, price_limit_provider
|
||
from app.services.price_limits import day_key
|
||
|
||
LEASE_KEY = 'price-limits:collector'
|
||
LEASE_SECONDS = 300
|
||
TARGET_CAP = 1200
|
||
|
||
|
||
def _summary(status, day, targets=(), entries=None, errors=(), requests=0):
|
||
entries = entries or {}
|
||
errors = list(errors) + [c+': '+entries[c]['last_error'] for c in targets
|
||
if entries.get(c, {}).get('last_error')]
|
||
valid = sum(entries.get(c, {}).get('status') == 'VALID' for c in targets)
|
||
return {'status': status, 'day': day, 'target_count': len(targets),
|
||
'valid_count': valid, 'missing_count': len(targets)-valid,
|
||
'error_count': len(errors), 'errors': list(errors), 'requests': requests}
|
||
|
||
|
||
def _acquire(owner, now):
|
||
with store.locked(LEASE_KEY, 'price_limit_lease', {}) as lease:
|
||
until = timestamp(lease.get('until'))
|
||
if until and until > now:
|
||
return False
|
||
lease.update(owner=owner, until=(now+timedelta(seconds=LEASE_SECONDS)).isoformat())
|
||
return True
|
||
|
||
|
||
def _release(owner):
|
||
with store.locked(LEASE_KEY, 'price_limit_lease', {}) as lease:
|
||
if lease.get('owner') == owner:
|
||
lease.pop('owner', None)
|
||
lease.pop('until', None)
|
||
|
||
|
||
def _save(owner, document, now):
|
||
# 租约锁与缓存锁同一事务;不把HTTP或目标读取放进事务。
|
||
with store.locked(LEASE_KEY, 'price_limit_lease', {}) as lease:
|
||
until = timestamp(lease.get('until'))
|
||
if lease.get('owner') != owner or not until or now >= until:
|
||
return False
|
||
with store.locked(day_key(document['trade_date']), 'price_limits', {}) as value:
|
||
value.clear()
|
||
value.update(document)
|
||
return True
|
||
|
||
|
||
def _config():
|
||
p = param_store
|
||
return {'timeout': p.get_int('PMS_LIMIT_FETCH_TIMEOUT_SEC', 5),
|
||
'budget': p.get_int('PMS_LIMIT_ROUND_BUDGET_SEC', 30),
|
||
'max_codes': p.get_int('PMS_LIMIT_ROUND_MAX_CODES', 25),
|
||
'interval': p.get_float('PMS_LIMIT_MIN_REQUEST_INTERVAL_SEC', 1.2),
|
||
'retry_base': p.get_int('PMS_LIMIT_RETRY_BASE_SEC', 60),
|
||
'retry_max': p.get_int('PMS_LIMIT_RETRY_MAX_SEC', 900),
|
||
'max_failures': p.get_int('PMS_LIMIT_MAX_FAILURES_PER_DAY', 32),
|
||
'refresh': p.get_int('PMS_LIMIT_REFRESH_SEC', 900)}
|
||
|
||
|
||
def _failure(entry, reason, now, cfg, quarantine=False):
|
||
# 连接故障保留此前有效内容与原获取时间;确切坏值才撤销可执行资格。
|
||
entry['failures'] = min(100, int(entry.get('failures') or 0)+1)
|
||
entry['last_attempt'] = now.isoformat()
|
||
entry['last_error'] = reason
|
||
wait = min(cfg['retry_max'], cfg['retry_base'] * (2 ** min(entry['failures']-1, 10)))
|
||
entry['next_try'] = (now+timedelta(seconds=wait)).isoformat()
|
||
if quarantine or entry.get('status') != 'VALID':
|
||
entry['status'] = 'INVALID' if quarantine else 'MISSING'
|
||
|
||
|
||
def _accept(entry, record, source, now, cfg):
|
||
old = entry.get('record') or {}
|
||
if old and old.get('version') != record['version']:
|
||
history = entry.setdefault('revisions', [])
|
||
history.append({'record': old, 'source': entry.get('source'),
|
||
'fetched_at': entry.get('fetched_at'), 'superseded_at': now.isoformat()})
|
||
entry['revisions'] = history[-5:]
|
||
entry.update(status='VALID', record=record, source=source, fetched_at=now.isoformat(),
|
||
last_attempt=now.isoformat(), last_error='', failures=0,
|
||
next_try=(now+timedelta(seconds=cfg['refresh'])).isoformat())
|
||
|
||
|
||
def _shared_rows(day, codes):
|
||
if not codes:
|
||
return []
|
||
params = {'day': day}
|
||
params.update({f'c{i}': c for i, c in enumerate(codes)})
|
||
names = ','.join(':'+k for k in params if k != 'day')
|
||
return db.fetch_all('SELECT ts_code,trade_date,pre_close,up_limit,down_limit FROM gp_stk_limit '
|
||
'WHERE trade_date=:day AND ts_code IN ('+names+')', params)
|
||
|
||
|
||
def _reserve_request(owner, now, interval):
|
||
"""持久化限速水位;重建进程也不会立刻重复请求。"""
|
||
with store.locked(LEASE_KEY, 'price_limit_lease', {}) as lease:
|
||
until = timestamp(lease.get('until'))
|
||
if lease.get('owner') != owner or not until or now >= until:
|
||
return None
|
||
last = timestamp(lease.get('next_request_at'))
|
||
if last and last > now:
|
||
return (last-now).total_seconds()
|
||
lease['next_request_at'] = (now+timedelta(seconds=interval)).isoformat()
|
||
return 0.0
|
||
|
||
|
||
def run_once(now=None):
|
||
"""只采集实际北京日;now仅供隔离测试注入时钟,无历史补数入口。"""
|
||
current = local_now(now)
|
||
day = current.date().isoformat()
|
||
if settings.PMS_TABLE_PREFIX:
|
||
return _summary('UNAUTHORIZED_INSTANCE', day)
|
||
if not param_store.get_bool('PMS_LIMIT_COLLECT_ENABLED', False):
|
||
return _summary('DISABLED', day)
|
||
if tradedays.calendar_degraded(current.date()) or not tradedays.is_trade_day(current.date()):
|
||
return _summary('CALENDAR_WAIT', day)
|
||
try:
|
||
start = datetime.strptime(param_store.get('PMS_LIMIT_COLLECT_START', '08:55'), '%H:%M').time()
|
||
end = datetime.strptime(param_store.get('PMS_LIMIT_COLLECT_END', '18:00'), '%H:%M').time()
|
||
except (TypeError, ValueError):
|
||
return _summary('INVALID_WINDOW', day)
|
||
if not start <= current.time().replace(tzinfo=None) <= end:
|
||
return _summary('OUTSIDE_WINDOW', day)
|
||
owner = uuid.uuid4().hex
|
||
if not _acquire(owner, current):
|
||
return _summary('BUSY', day)
|
||
started = time.monotonic()
|
||
# 实际墙钟由单调时钟增量推进,测试也能准确覆盖跨日、租约和超时。
|
||
clock = lambda: current+timedelta(seconds=time.monotonic()-started)
|
||
try:
|
||
cfg = _config()
|
||
if not (1 <= cfg['timeout'] <= 10 and 5 <= cfg['budget'] <= 60 and
|
||
1 <= cfg['max_codes'] <= 60 and 1 <= cfg['interval'] <= 10 and
|
||
30 <= cfg['retry_base'] <= 600 and cfg['retry_base'] <= cfg['retry_max'] <= 3600 and
|
||
1 <= cfg['max_failures'] <= 100 and 300 <= cfg['refresh'] <= 7200):
|
||
return _summary('INVALID_CONFIG', day)
|
||
target_info = price_limit_targets.collect_targets(day)
|
||
all_codes = sorted({c for c in target_info['codes'] if core.supported(c)})
|
||
codes, errors = all_codes[:TARGET_CAP], list(target_info['errors'])
|
||
if len(all_codes) > TARGET_CAP:
|
||
errors.append('采集目标超过1200只,未覆盖部分保持等待')
|
||
previous, _ = store.get(day_key(day))
|
||
document = copy.deepcopy(previous) if isinstance(previous, dict) and previous.get('trade_date') == day else {}
|
||
document.update(trade_date=day, schema_version=1, targets=codes,
|
||
target_sources={c: target_info['sources'].get(c, []) for c in codes})
|
||
entries = document.get('entries')
|
||
if not isinstance(entries, dict):
|
||
entries = document['entries'] = {}
|
||
# 再校验已存值,损坏或异日内容不能靠VALID标记继续使用。
|
||
for code, entry in list(entries.items()):
|
||
if not core.supported(code) or not isinstance(entry, dict):
|
||
entries.pop(code, None)
|
||
continue
|
||
if type(entry.get('failures')) is not int or not 0 <= entry['failures'] <= 100:
|
||
entry['failures'] = 0
|
||
for field in ('next_try', 'last_attempt'):
|
||
stamp = timestamp(entry.get(field))
|
||
if not stamp or stamp.date().isoformat() != day:
|
||
entry.pop(field, None)
|
||
else:
|
||
entry[field] = stamp.isoformat()
|
||
if not isinstance(entry.get('revisions'), list):
|
||
entry['revisions'] = []
|
||
for field in ('last_error', 'shared_invalid_reason'):
|
||
reason = entry.get(field)
|
||
if reason and (not isinstance(reason, str) or not re.fullmatch('[A-Z_]{1,64}', reason)):
|
||
entry[field] = 'CACHE_META_INVALID'
|
||
if entry.get('status') == 'VALID':
|
||
checked = core.normalize_rows([entry.get('record')], day, [code])['records'].get(code)
|
||
if (not checked or checked['version'] != (entry.get('record') or {}).get('version')
|
||
or entry.get('source') not in ('gp_stk_limit', 'tushare.stk_limit')):
|
||
entry.update(status='INVALID', last_error='CACHE_INVALID', next_try=current.isoformat())
|
||
for code in codes:
|
||
entries.setdefault(code, {'status': 'MISSING', 'failures': 0})
|
||
try:
|
||
shared = core.normalize_rows(_shared_rows(day, codes), day, codes) if codes else {'records': {}, 'invalid': {}, 'errors': []}
|
||
errors.extend(shared['errors'])
|
||
for code, record in shared['records'].items():
|
||
entry = entries[code]
|
||
if entry.get('source') == 'tushare.stk_limit':
|
||
continue
|
||
if entry.get('status') != 'VALID' or (entry.get('record') or {}).get('version') != record['version']:
|
||
_accept(entry, record, 'gp_stk_limit', clock(), cfg)
|
||
entry.pop('shared_invalid_reason', None)
|
||
for code, reason in shared['invalid'].items():
|
||
if code in entries and entries[code].get('source') != 'tushare.stk_limit':
|
||
entry = entries[code]
|
||
# 相同坏值不在每轮重置退避;第一次仍允许直接向Tushare补齐。
|
||
if entry.get('shared_invalid_reason') != reason:
|
||
entry.update(status='INVALID', last_error=reason, next_try=current.isoformat())
|
||
entry['shared_invalid_reason'] = reason
|
||
except Exception:
|
||
errors.append('共享限制价读取失败')
|
||
# 没有采到过的票先查,避免前排失败长时间挤占后排。每票退避独立。
|
||
due = [c for c in codes if int(entries[c].get('failures') or 0) < cfg['max_failures']
|
||
and (timestamp(entries[c].get('next_try')) or current) <= clock()]
|
||
due.sort(key=lambda c: (entries[c].get('status') == 'VALID', entries[c].get('last_attempt') or '', c))
|
||
deadline = time.monotonic()+cfg['budget']
|
||
count = 0
|
||
if not settings.PMS_TUSHARE_TOKEN and due:
|
||
errors.append('采集密钥未配置')
|
||
else:
|
||
for code in due:
|
||
left = deadline-time.monotonic()
|
||
if count >= cfg['max_codes'] or left < 1 or clock().date().isoformat() != day:
|
||
break
|
||
delay = _reserve_request(owner, clock(), cfg['interval'])
|
||
if delay is None:
|
||
return _summary('LEASE_LOST', day, codes, entries, errors, count)
|
||
if delay:
|
||
if delay+1 > left:
|
||
break
|
||
time.sleep(delay)
|
||
delay = _reserve_request(owner, clock(), cfg['interval'])
|
||
if delay is None or delay > 0:
|
||
break
|
||
result = price_limit_provider.fetch_limit(code, day, settings.PMS_TUSHARE_TOKEN,
|
||
timeout_sec=min(cfg['timeout'], max(1, deadline-time.monotonic())))
|
||
count += 1
|
||
at = clock()
|
||
if at.date().isoformat() != day or time.monotonic() > deadline:
|
||
_failure(entries[code], 'ROUND_TIMEOUT', at, cfg)
|
||
errors.append('本轮采集超时,迟到响应未采纳')
|
||
break
|
||
entry = entries[code]
|
||
if code in result['records']:
|
||
_accept(entry, result['records'][code], 'tushare.stk_limit', at, cfg)
|
||
else:
|
||
invalid = result.get('invalid') or {}
|
||
reason = invalid.get(code) or (result['errors'][0] if result['errors'] else ('UNEXPECTED_SECURITY' if invalid else 'NOT_PUBLISHED'))
|
||
_failure(entry, reason, at, cfg, quarantine=bool(invalid))
|
||
end_at = clock()
|
||
summary = _summary('COMPLETE' if not errors and all(entries[c].get('status') == 'VALID' and not entries[c].get('last_error') for c in codes) else 'PARTIAL',
|
||
day, codes, entries, errors, count)
|
||
document.update(last_round_at=end_at.isoformat(), last_summary=summary)
|
||
if end_at.date().isoformat() != day or not _save(owner, document, end_at):
|
||
return _summary('LEASE_LOST', day, codes, entries, errors, count)
|
||
return summary
|
||
finally:
|
||
_release(owner)
|