"""ETF观测契约、持久化/ACK次序和信号补显。全为合成数据及内存替身。""" import unit_test_guard import contextlib import copy from datetime import datetime,timedelta import hashlib import json import sys from pathlib import Path from unittest import TestCase, main from unittest.mock import patch sys.path.insert(0,str(Path(__file__).resolve().parents[1])) from app.services import etf_observation as svc, upstream_signals as upstream, param_store from app.core.etf_contract import consumer as contract from app.core.etf_contract.timing import DELAYED_POLICY, lease_until, admission_until from config.settings import settings NOW=datetime.fromisoformat('2026-11-05T10:03:00+08:00') DAY=NOW.date().isoformat() def fixture(hm='10:00',seq=1,state='OPEN',revision=0): end=datetime.fromisoformat(DAY+'T'+hm+':00+08:00'); published=end+timedelta(minutes=2) observed={'amount_unit':'CNY','flow_basis':'TURNOVER_NOT_NET_FLOW','amount':100000000.,'cumulative_amount':200000000.,'window_amount_status':'OK','cumulative_amount_status':'OK','full_day_median20':100000000.,'cumulative_to_full_day_median20':2.} metrics={'threshold_version':'test-threshold','amount':100000000.,'cumulative_amount':200000000.,'pressure_score':.1} block={'data_status':'OK','anomaly':True,'direction':'MIXED','direction_status':'OK','severity':1,'metrics':metrics} snap={'schema_version':1,'round_id':'20261105T'+hm.replace(':',''),'trade_date':DAY,'revision':revision,'window_end':end.isoformat(),'published_at':published.isoformat(),'mode':'LIVE','publication_mode':'observation','model_status':'uncalibrated','calibration_status':'PROVISIONAL','threshold_version':'test-threshold','model_version':'synthetic','data_status':'OK','session_closed':hm=='15:00','timing_policy_version':DELAYED_POLICY,'observation_contract_version':'etf-observation-1.0','purpose':'MARKET_OBSERVATION','trade_instruction':False,'release_id':'test-release','effective_from':DAY+'T09:45:00+08:00','review_status':'ACCUMULATING','calibration_review':{'sample_count':33,'target_days':60,'eligible_cohort_size':1,'by_subject':{'MARKET:BROAD_ETF':33}},'subjects':[{'subject_type':'MARKET','subject_id':'BROAD_ETF','data_status':'OK','member_count':1,'observation':observed,'WINDOW_15M':block,'SESSION_CUM':copy.deepcopy(block)}]} event={k:copy.deepcopy(snap[k]) for k in ('schema_version','round_id','trade_date','revision','window_end','published_at','model_version','timing_policy_version')+contract.OBSERVATION_EVENT_FIELDS} event.update(event_id='synthetic-event',event_seq=seq,event_state=state,subject_type='MARKET',subject_id='BROAD_ETF',scope='WINDOW_15M',producer_id='mtf_etf',flow_basis='price_volume_proxy',window_start=(end-timedelta(minutes=15)).isoformat(),valid_until=lease_until(end,DELAYED_POLICY).isoformat(),opened_at=DAY+'T10:00:00+08:00',direction='MIXED',direction_status='OK',last_known_direction='MIXED',severity=1,metrics=metrics,reason_codes=[]) raw=json.dumps(snap,ensure_ascii=False) mf={k:copy.deepcopy(snap[k]) for k in ('schema_version','round_id','trade_date','revision','window_end','published_at','mode','publication_mode','session_closed','timing_policy_version')+contract.OBSERVATION_FIELDS} suffix=f"{DAY}:{snap['round_id']}:r{revision}" mf.update(status='READY',snapshot_sha256=hashlib.sha256(raw.encode()).hexdigest(),manifest_key=svc.NS+':manifest:'+suffix,snapshot_key=svc.NS+':snapshot:'+suffix,live_eligible=hm!='15:00',valid_until=lease_until(end,DELAYED_POLICY).isoformat(),admission_until=admission_until(end,DELAYED_POLICY).isoformat()) return event,mf,raw class MemoryStore: def __init__(self):self.docs={};self.fail=False def get(self,k):return copy.deepcopy(self.docs.get(k)),1 def put_once(self,k,kind,v): if self.fail:raise RuntimeError('database unavailable') if k in self.docs and self.docs[k]!=v:raise ValueError('immutable') self.docs[k]=copy.deepcopy(v) @contextlib.contextmanager def locked(self,k,kind,initial): if self.fail:raise RuntimeError('database unavailable') before=copy.deepcopy(self.docs);v=copy.deepcopy(self.docs.get(k,initial)) try:yield v except BaseException:self.docs=before;raise else:self.docs[k]=v class RedisFake: def __init__(self,store=None):self.values={};self.rows=[];self.pending=[];self.acks=[];self.store=store def get(self,key):return self.values.get(key) def strlen(self,key):return len(self.values.get(key,'').encode()) def getrange(self,key,a,b):return self.values.get(key,'').encode()[a:b+1].decode() def xgroup_create(self,*a,**kw):pass def scan_iter(self,**kw):return [] def xreadgroup(self,group,consumer,streams,count): if next(iter(streams.values()))=='0-0': rows=list(self.pending) else:rows=self.rows[:count];self.rows=self.rows[count:];self.pending+=rows return [(next(iter(streams)),rows)] if rows else [] def xack(self,key,group,sid): assert any(k.endswith(':'+sid) for k in self.store.docs),'ACK before durable record' self.acks.append(sid);self.pending=[r for r in self.pending if r[0]!=sid] class ETFTests(TestCase): def setUp(self): self.stack=contextlib.ExitStack();self.addCleanup(self.stack.close) self.store=MemoryStore();self.stack.enter_context(patch.object(svc,'store',self.store)) self.stack.enter_context(patch.object(param_store,'get_bool',return_value=True)) self.snap,self.bus=RedisFake(),RedisFake(self.store) def load(self,hm='10:00',seq=1,state='OPEN',revision=0,sid=None): e,m,r=fixture(hm,seq,state,revision);sid=sid or str(seq)+'-0' self.bus.rows.append((sid,{'data':json.dumps(e)}));self.bus.values[m['manifest_key']]=json.dumps(m);self.snap.values[m['snapshot_key']]=r;self.snap.values[svc.NS+':latest']=json.dumps(m) return e,m,r def runat(self,at=NOW):return svc.consume_once(self.snap,self.bus,clock=lambda:at) def state(self):return self.store.docs[svc.day_key(DAY)] def test_original_valid_envelope_saved_before_ack(self): self.load();self.assertEqual(self.runat()['status'],'ok');self.assertEqual(self.bus.acks,['1-0']);self.assertEqual(len(self.state()['events']),1) def test_duplicate_delivery_and_restart_do_not_duplicate(self): self.load();self.runat();first=copy.deepcopy(self.state()['events']);self.load();self.runat();self.assertEqual(self.state()['events'],first) def test_database_failure_keeps_pending(self): self.load();self.store.fail=True with self.assertRaises(RuntimeError):self.runat() self.assertFalse(self.bus.acks);self.assertTrue(self.bus.pending) self.store.fail=False;self.runat();self.assertEqual(self.bus.acks,['1-0']) def test_current_continue_updates_one_row(self): self.load();self.runat();self.load('10:15',2,'CONTINUE');self.runat(NOW+timedelta(minutes=15));self.assertEqual(len(self.state()['events']),1);self.assertEqual(self.state()['events']['synthetic-event']['event']['event_seq'],2) def test_older_revision_does_not_replace(self): self.load(revision=1);self.runat();self.load(seq=2,revision=0);self.runat();self.assertEqual(self.state()['summary']['revision'],1);self.assertEqual(self.state()['events']['synthetic-event']['event']['revision'],1) def test_late_open_is_archived_not_current(self): self.load();self.runat(NOW+timedelta(minutes=31));self.assertFalse(self.state()['events']);self.assertEqual(len(self.bus.acks),1) def test_late_terminal_closes_previously_open(self): self.load();self.runat();self.load('10:15',2,'RESOLVED');self.runat(NOW+timedelta(hours=1));self.assertEqual(self.state()['events']['synthetic-event']['event']['event_state'],'RESOLVED') def test_sequence_gap_is_exposed(self): self.load(seq=3,state='CONTINUE');self.runat();self.assertTrue(self.state()['sequence_gap']) def test_corrupt_hash_not_acknowledged(self): e,m,r=self.load();self.snap.values[m['snapshot_key']]=r+' ';self.assertEqual(self.runat()['status'],'unavailable');self.assertFalse(self.bus.acks) def test_bad_pending_does_not_block_later_event_or_latest_snapshot(self): e,m,r=self.load();del self.snap.values[m['snapshot_key']] self.load('10:15',2,'CONTINUE') self.assertEqual(self.runat(NOW+timedelta(minutes=15))['status'],'unavailable') self.assertEqual(self.bus.acks,['2-0']);self.assertEqual(self.state()['summary']['round_id'],'20261105T1015') def test_other_subject_eligible_does_not_make_market_live(self): self.load();self.runat();state=copy.deepcopy(self.state()) for permission in state['summary']['scopes'].values():permission['observation_eligible']=False self.assertFalse(svc.presentation(state,NOW)['current_eligible']) def test_source_recovery_reprocesses_pending(self): e,m,r=self.load();del self.snap.values[m['snapshot_key']];self.runat();self.snap.values[m['snapshot_key']]=r;self.runat();self.assertEqual(self.bus.acks,['1-0']) def test_no_events_still_publishes_summary(self): self.load();self.bus.rows=[];self.runat();self.assertEqual(self.state()['summary']['market']['observation']['amount'],100000000.) def test_read_clock_after_fetch_prevents_expired_activation(self): self.load();ticks=iter([NOW,NOW+timedelta(hours=1),NOW+timedelta(hours=1),NOW+timedelta(hours=1),NOW+timedelta(hours=1)]) svc.consume_once(self.snap,self.bus,clock=lambda:next(ticks));self.assertFalse(self.state()['events']) def test_expiry_is_recomputed_without_new_messages(self): self.load();self.runat();view=svc.presentation(self.state(),NOW+timedelta(minutes=28));self.assertTrue(view['events'][0]['stale']);self.assertFalse(view['current_eligible']) def test_read_failure_preserves_valid_cached_facts(self): self.load();self.runat();state=self.state();state['error']='ConnectionError';view=svc.presentation(state,NOW);self.assertEqual(view['status'],'unavailable');self.assertTrue(view['current_eligible']);self.assertFalse(view['trade_instruction']) def test_secrets_never_enter_parameter_registry(self): for key in ('PMS_ETF_SNAPSHOT_REDIS_URL','PMS_ETF_EVENT_REDIS_URL'): self.assertIn(key,param_store.SECRET_KEYS);self.assertNotIn(key,param_store._editable_keys()) def test_disallowed_subject_and_unknown_direction(self): self.assertFalse(svc.subject_allowed('ETF','920001.BJ'));self.assertFalse(svc.subject_allowed('INDEX','bad')) def test_vendor_bytes_match_pinned_source(self): path=Path(contract.__file__).parent;manifest=json.loads((path/'PROVENANCE.json').read_text()) for name,digest in manifest['sha256'].items():self.assertEqual(hashlib.sha256((path/name).read_bytes()).hexdigest(),digest) class FocusTests(TestCase): def setUp(self): upstream._focused_streams.clear() p=patch.object(param_store,'get_int',side_effect=lambda key,default=0:default);p.start();self.addCleanup(p.stop) def test_old_watched_and_held_rows_survive_global_limit_and_are_cached(self): rows=[(str(1000-i)+'-0',{'ts_code':'600009.SH'})for i in range(70)]+[('800-0',{'ts_code':'300001.SZ'}),('799-0',{'ts_code':'600001.SH'})] with patch.object(upstream,'_focus_codes',return_value={'300001.SZ','600001.SH'}),patch.object(upstream,'_scan_stream',return_value=(rows,False)) as scan: got=upstream._entries(None,'test',40);self.assertEqual(len(got),42);self.assertEqual(upstream._entries(None,'test',40),got);self.assertEqual(scan.call_count,1) def test_metrics_use_original_stream_time_not_refresh_time(self): ms=int(NOW.timestamp()*1000) with patch.object(upstream,'_entries',return_value=[(str(ms)+'-0',{'ts_code':'SZ300001','data':json.dumps({'direction':'inflow','z_dd':6})})]): row=upstream._read_metrics(None)['items'][0];self.assertEqual(row['ts'],ms);self.assertEqual(row['id'],str(ms)+'-0');self.assertEqual(row['ts_from'],'stream_id') def test_empty_source_creates_no_placeholder(self): with patch.object(upstream,'_entries',return_value=[]):self.assertEqual(upstream._read_metrics(None)['items'],[]) def test_focus_removed_no_longer_reused(self): rows=[('1-0',{'ts_code':'300001.SZ'})] with patch.object(upstream,'_focus_codes',return_value={'300001.SZ'}),patch.object(upstream,'_scan_stream',return_value=(rows,False)):self.assertEqual(len(upstream._entries(None,'test',0)),1) with patch.object(upstream,'_focus_codes',return_value={'600001.SH'}),patch.object(upstream,'_scan_stream',return_value=(rows,False)):self.assertEqual(upstream._entries(None,'test',0),[]) if __name__=='__main__':main()