tradingSystem/scripts/test_etf_observation.py

147 lines
12 KiB
Python

"""ETF观测契约、持久化/ACK次序和信号补显。全为合成数据及内存替身。"""
import unit_test_guard
import contextlib
import copy
from datetime import datetime,timedelta
import hashlib
import json
import sys
from pathlib import Path
from unittest import TestCase, main
from unittest.mock import patch
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from app.services import etf_observation as svc, upstream_signals as upstream, param_store
from app.core.etf_contract import consumer as contract
from app.core.etf_contract.timing import DELAYED_POLICY, lease_until, admission_until
from config.settings import settings
NOW=datetime.fromisoformat('2026-11-05T10:03:00+08:00')
DAY=NOW.date().isoformat()
def fixture(hm='10:00',seq=1,state='OPEN',revision=0):
end=datetime.fromisoformat(DAY+'T'+hm+':00+08:00'); published=end+timedelta(minutes=2)
observed={'amount_unit':'CNY','flow_basis':'TURNOVER_NOT_NET_FLOW','amount':100000000.,'cumulative_amount':200000000.,'window_amount_status':'OK','cumulative_amount_status':'OK','full_day_median20':100000000.,'cumulative_to_full_day_median20':2.}
metrics={'threshold_version':'test-threshold','amount':100000000.,'cumulative_amount':200000000.,'pressure_score':.1}
block={'data_status':'OK','anomaly':True,'direction':'MIXED','direction_status':'OK','severity':1,'metrics':metrics}
snap={'schema_version':1,'round_id':'20261105T'+hm.replace(':',''),'trade_date':DAY,'revision':revision,'window_end':end.isoformat(),'published_at':published.isoformat(),'mode':'LIVE','publication_mode':'observation','model_status':'uncalibrated','calibration_status':'PROVISIONAL','threshold_version':'test-threshold','model_version':'synthetic','data_status':'OK','session_closed':hm=='15:00','timing_policy_version':DELAYED_POLICY,'observation_contract_version':'etf-observation-1.0','purpose':'MARKET_OBSERVATION','trade_instruction':False,'release_id':'test-release','effective_from':DAY+'T09:45:00+08:00','review_status':'ACCUMULATING','calibration_review':{'sample_count':33,'target_days':60,'eligible_cohort_size':1,'by_subject':{'MARKET:BROAD_ETF':33}},'subjects':[{'subject_type':'MARKET','subject_id':'BROAD_ETF','data_status':'OK','member_count':1,'observation':observed,'WINDOW_15M':block,'SESSION_CUM':copy.deepcopy(block)}]}
event={k:copy.deepcopy(snap[k]) for k in ('schema_version','round_id','trade_date','revision','window_end','published_at','model_version','timing_policy_version')+contract.OBSERVATION_EVENT_FIELDS}
event.update(event_id='synthetic-event',event_seq=seq,event_state=state,subject_type='MARKET',subject_id='BROAD_ETF',scope='WINDOW_15M',producer_id='mtf_etf',flow_basis='price_volume_proxy',window_start=(end-timedelta(minutes=15)).isoformat(),valid_until=lease_until(end,DELAYED_POLICY).isoformat(),opened_at=DAY+'T10:00:00+08:00',direction='MIXED',direction_status='OK',last_known_direction='MIXED',severity=1,metrics=metrics,reason_codes=[])
raw=json.dumps(snap,ensure_ascii=False)
mf={k:copy.deepcopy(snap[k]) for k in ('schema_version','round_id','trade_date','revision','window_end','published_at','mode','publication_mode','session_closed','timing_policy_version')+contract.OBSERVATION_FIELDS}
suffix=f"{DAY}:{snap['round_id']}:r{revision}"
mf.update(status='READY',snapshot_sha256=hashlib.sha256(raw.encode()).hexdigest(),manifest_key=svc.NS+':manifest:'+suffix,snapshot_key=svc.NS+':snapshot:'+suffix,live_eligible=hm!='15:00',valid_until=lease_until(end,DELAYED_POLICY).isoformat(),admission_until=admission_until(end,DELAYED_POLICY).isoformat())
return event,mf,raw
class MemoryStore:
def __init__(self):self.docs={};self.fail=False
def get(self,k):return copy.deepcopy(self.docs.get(k)),1
def put_once(self,k,kind,v):
if self.fail:raise RuntimeError('database unavailable')
if k in self.docs and self.docs[k]!=v:raise ValueError('immutable')
self.docs[k]=copy.deepcopy(v)
@contextlib.contextmanager
def locked(self,k,kind,initial):
if self.fail:raise RuntimeError('database unavailable')
before=copy.deepcopy(self.docs);v=copy.deepcopy(self.docs.get(k,initial))
try:yield v
except BaseException:self.docs=before;raise
else:self.docs[k]=v
class RedisFake:
def __init__(self,store=None):self.values={};self.rows=[];self.pending=[];self.acks=[];self.store=store
def get(self,key):return self.values.get(key)
def strlen(self,key):return len(self.values.get(key,'').encode())
def getrange(self,key,a,b):return self.values.get(key,'').encode()[a:b+1].decode()
def xgroup_create(self,*a,**kw):pass
def scan_iter(self,**kw):return []
def xreadgroup(self,group,consumer,streams,count):
if next(iter(streams.values()))=='0-0': rows=list(self.pending)
else:rows=self.rows[:count];self.rows=self.rows[count:];self.pending+=rows
return [(next(iter(streams)),rows)] if rows else []
def xack(self,key,group,sid):
assert any(k.endswith(':'+sid) for k in self.store.docs),'ACK before durable record'
self.acks.append(sid);self.pending=[r for r in self.pending if r[0]!=sid]
class ETFTests(TestCase):
def setUp(self):
self.stack=contextlib.ExitStack();self.addCleanup(self.stack.close)
self.store=MemoryStore();self.stack.enter_context(patch.object(svc,'store',self.store))
self.stack.enter_context(patch.object(param_store,'get_bool',return_value=True))
self.snap,self.bus=RedisFake(),RedisFake(self.store)
def load(self,hm='10:00',seq=1,state='OPEN',revision=0,sid=None):
e,m,r=fixture(hm,seq,state,revision);sid=sid or str(seq)+'-0'
self.bus.rows.append((sid,{'data':json.dumps(e)}));self.bus.values[m['manifest_key']]=json.dumps(m);self.snap.values[m['snapshot_key']]=r;self.snap.values[svc.NS+':latest']=json.dumps(m)
return e,m,r
def runat(self,at=NOW):return svc.consume_once(self.snap,self.bus,clock=lambda:at)
def state(self):return self.store.docs[svc.day_key(DAY)]
def test_original_valid_envelope_saved_before_ack(self):
self.load();self.assertEqual(self.runat()['status'],'ok');self.assertEqual(self.bus.acks,['1-0']);self.assertEqual(len(self.state()['events']),1)
def test_duplicate_delivery_and_restart_do_not_duplicate(self):
self.load();self.runat();first=copy.deepcopy(self.state()['events']);self.load();self.runat();self.assertEqual(self.state()['events'],first)
def test_database_failure_keeps_pending(self):
self.load();self.store.fail=True
with self.assertRaises(RuntimeError):self.runat()
self.assertFalse(self.bus.acks);self.assertTrue(self.bus.pending)
self.store.fail=False;self.runat();self.assertEqual(self.bus.acks,['1-0'])
def test_current_continue_updates_one_row(self):
self.load();self.runat();self.load('10:15',2,'CONTINUE');self.runat(NOW+timedelta(minutes=15));self.assertEqual(len(self.state()['events']),1);self.assertEqual(self.state()['events']['synthetic-event']['event']['event_seq'],2)
def test_older_revision_does_not_replace(self):
self.load(revision=1);self.runat();self.load(seq=2,revision=0);self.runat();self.assertEqual(self.state()['summary']['revision'],1);self.assertEqual(self.state()['events']['synthetic-event']['event']['revision'],1)
def test_late_open_is_archived_not_current(self):
self.load();self.runat(NOW+timedelta(minutes=31));self.assertFalse(self.state()['events']);self.assertEqual(len(self.bus.acks),1)
def test_late_terminal_closes_previously_open(self):
self.load();self.runat();self.load('10:15',2,'RESOLVED');self.runat(NOW+timedelta(hours=1));self.assertEqual(self.state()['events']['synthetic-event']['event']['event_state'],'RESOLVED')
def test_sequence_gap_is_exposed(self):
self.load(seq=3,state='CONTINUE');self.runat();self.assertTrue(self.state()['sequence_gap'])
def test_corrupt_hash_not_acknowledged(self):
e,m,r=self.load();self.snap.values[m['snapshot_key']]=r+' ';self.assertEqual(self.runat()['status'],'unavailable');self.assertFalse(self.bus.acks)
def test_bad_pending_does_not_block_later_event_or_latest_snapshot(self):
e,m,r=self.load();del self.snap.values[m['snapshot_key']]
self.load('10:15',2,'CONTINUE')
self.assertEqual(self.runat(NOW+timedelta(minutes=15))['status'],'unavailable')
self.assertEqual(self.bus.acks,['2-0']);self.assertEqual(self.state()['summary']['round_id'],'20261105T1015')
def test_other_subject_eligible_does_not_make_market_live(self):
self.load();self.runat();state=copy.deepcopy(self.state())
for permission in state['summary']['scopes'].values():permission['observation_eligible']=False
self.assertFalse(svc.presentation(state,NOW)['current_eligible'])
def test_source_recovery_reprocesses_pending(self):
e,m,r=self.load();del self.snap.values[m['snapshot_key']];self.runat();self.snap.values[m['snapshot_key']]=r;self.runat();self.assertEqual(self.bus.acks,['1-0'])
def test_no_events_still_publishes_summary(self):
self.load();self.bus.rows=[];self.runat();self.assertEqual(self.state()['summary']['market']['observation']['amount'],100000000.)
def test_read_clock_after_fetch_prevents_expired_activation(self):
self.load();ticks=iter([NOW,NOW+timedelta(hours=1),NOW+timedelta(hours=1),NOW+timedelta(hours=1),NOW+timedelta(hours=1)])
svc.consume_once(self.snap,self.bus,clock=lambda:next(ticks));self.assertFalse(self.state()['events'])
def test_expiry_is_recomputed_without_new_messages(self):
self.load();self.runat();view=svc.presentation(self.state(),NOW+timedelta(minutes=28));self.assertTrue(view['events'][0]['stale']);self.assertFalse(view['current_eligible'])
def test_read_failure_preserves_valid_cached_facts(self):
self.load();self.runat();state=self.state();state['error']='ConnectionError';view=svc.presentation(state,NOW);self.assertEqual(view['status'],'unavailable');self.assertTrue(view['current_eligible']);self.assertFalse(view['trade_instruction'])
def test_secrets_never_enter_parameter_registry(self):
for key in ('PMS_ETF_SNAPSHOT_REDIS_URL','PMS_ETF_EVENT_REDIS_URL'):
self.assertIn(key,param_store.SECRET_KEYS);self.assertNotIn(key,param_store._editable_keys())
def test_disallowed_subject_and_unknown_direction(self):
self.assertFalse(svc.subject_allowed('ETF','920001.BJ'));self.assertFalse(svc.subject_allowed('INDEX','bad'))
def test_vendor_bytes_match_pinned_source(self):
path=Path(contract.__file__).parent;manifest=json.loads((path/'PROVENANCE.json').read_text())
for name,digest in manifest['sha256'].items():self.assertEqual(hashlib.sha256((path/name).read_bytes()).hexdigest(),digest)
class FocusTests(TestCase):
def setUp(self):
upstream._focused_streams.clear()
p=patch.object(param_store,'get_int',side_effect=lambda key,default=0:default);p.start();self.addCleanup(p.stop)
def test_old_watched_and_held_rows_survive_global_limit_and_are_cached(self):
rows=[(str(1000-i)+'-0',{'ts_code':'600009.SH'})for i in range(70)]+[('800-0',{'ts_code':'300001.SZ'}),('799-0',{'ts_code':'600001.SH'})]
with patch.object(upstream,'_focus_codes',return_value={'300001.SZ','600001.SH'}),patch.object(upstream,'_scan_stream',return_value=(rows,False)) as scan:
got=upstream._entries(None,'test',40);self.assertEqual(len(got),72);self.assertEqual(upstream._entries(None,'test',40),got);self.assertEqual(scan.call_count,1)
def test_metrics_use_original_stream_time_not_refresh_time(self):
ms=int(NOW.timestamp()*1000)
with patch.object(upstream,'_entries',return_value=[(str(ms)+'-0',{'ts_code':'SZ300001','data':json.dumps({'direction':'inflow','z_dd':6})})]):
row=upstream._read_metrics(None)['items'][0];self.assertEqual(row['ts'],ms);self.assertEqual(row['id'],str(ms)+'-0');self.assertEqual(row['ts_from'],'stream_id')
def test_empty_source_creates_no_placeholder(self):
with patch.object(upstream,'_entries',return_value=[]):self.assertEqual(upstream._read_metrics(None)['items'],[])
def test_focus_changes_only_grouping_not_source_coverage(self):
rows=[('1-0',{'ts_code':'300001.SZ'})]
with patch.object(upstream,'_focus_codes',return_value={'300001.SZ'}),patch.object(upstream,'_scan_stream',return_value=(rows,False)):self.assertEqual(len(upstream._entries(None,'test',0)),1)
with patch.object(upstream,'_focus_codes',return_value={'600001.SH'}),patch.object(upstream,'_scan_stream',return_value=(rows,False)):self.assertEqual(len(upstream._entries(None,'test',0)),1)
if __name__=='__main__':main()