akg-factor-bridge/test_market_scope.py

191 lines
13 KiB
Python

"""沪深输入与冻结边界:全部虚构数据,离线读写临时文件与 SQLite。"""
import contextlib
import json
import re
import socket
import sqlite3
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
import pandas as pd
import common
import factors
import freeze
import market_scope as scope
DAY = '2026-11-04'
BAD = ['430001', '830001.BJ', 'BJ920001', '920001', '600001.SZ', 'SH000001', '510001.SH', '399001.SZ', '600001x', None]
def frame(codes, values=None):
return pd.DataFrame({'trade_date':[DAY]*len(codes), 'stock_code':codes,
'factor_value':values or list(range(len(codes)))})
class MarketScopeTests(unittest.TestCase):
def setUp(self):
p=patch.object(socket.socket, 'connect', side_effect=AssertionError('禁止联网'));p.start();self.addCleanup(p.stop)
for name in ('read_pg', 'read_mysql', 'factor_conn'):
p=patch.object(common.db, name, side_effect=AssertionError('必须替换数据库依赖'));p.start();self.addCleanup(p.stop)
def test_normalizes_prefix_suffix_and_bare_code(self):
for variants, expected in [(['SH600001','600001.SH','600001',' sh600001 '],'600001.SH'),
(['SZ000001','000001.SZ','000001'],'000001.SZ'),
(['688001','SH688001'],'688001.SH'),
(['301001','301001.SZ'],'301001.SZ')]:
for value in variants:
self.assertEqual(scope.normalize(value), expected)
self.assertEqual(scope.normalize(value,prefix=True), expected[-2:]+expected[:6])
for value in BAD: self.assertIsNone(scope.normalize(value), value)
def test_sql_and_python_accept_identical_scope(self):
sql = scope.sql_filter('stock_code')
pattern=sql.split("REGEXP '",1)[1][:-1]
for value in ['SH600001','600001.SH','600001','000001','SZ301001','688001.SH']+BAD:
self.assertEqual(bool(re.fullmatch(pattern,str(value).strip().upper())), scope.normalize(value) is not None, value)
self.assertIn(' ~ ',scope.sql_filter('ts_code',postgres=True))
with self.assertRaises(ValueError): scope.sql_filter('stock_code;DROP TABLE x')
def test_market_mode_never_includes_out_of_scope_or_mutates_source(self):
original=frame(['600001','SZ000001']+BAD); before=original.copy(deep=True)
with patch.object(common.config,'SUBFACTOR_UNIVERSE','market'):
result=common.universe_filter(original)
self.assertEqual(list(result.stock_code),['SH600001','SZ000001'])
pd.testing.assert_frame_equal(before,original)
def test_pool_mode_normalizes_both_sides_before_membership(self):
with patch.object(common.config,'SUBFACTOR_UNIVERSE','pool'), patch.object(common.db,'read_pg',return_value=pd.DataFrame({'ts_code':['SH600001','000001','830001.BJ']})) as read:
result=common.universe_filter(frame(['600001.SH','SZ000001','300001','BJ830001']))
self.assertEqual(list(result.stock_code),['SH600001','SZ000001'])
self.assertIn('WHERE',read.call_args[0][0]);self.assertIn(' ~ ',read.call_args[0][0])
def test_column_probe_does_not_read_arbitrary_security(self):
with patch.object(common.db,'read_mysql',return_value=pd.DataFrame()) as read:
self.assertEqual(factors._price_code_col(),common.config.PRICE_CODE_COL)
self.assertTrue(read.call_args[0][1].endswith('LIMIT 0'))
def test_heat_latest_batch_read_and_output_are_scoped(self):
calls=[]
def read(which,sql,params):
calls.append(sql)
self.assertIn('REGEXP',sql);self.assertNotIn('JOIN',sql)
if 'MAX(batch_no)' in sql: return pd.DataFrame({'trade_date':[DAY],'batch_no':[2]})
return pd.DataFrame({'trade_date':[DAY]*3,'stock_code':['600001.SH','SZ000001','BJ830001'],'batch_no':[2]*3,'score':[.1,.2,999]})
with patch.object(common.db,'read_mysql',side_effect=read),patch.object(common.config,'SUBFACTOR_UNIVERSE','market'):
result=factors.build_heat(DAY,DAY)
self.assertEqual(list(result.stock_code),['SH600001','SZ000001']);self.assertEqual(len(calls),2)
def test_heat_empty_qualified_batch_stays_empty(self):
with patch.object(common.db,'read_mysql',return_value=pd.DataFrame(columns=['trade_date','batch_no'])) as read:
self.assertTrue(factors.build_heat(DAY,DAY).empty)
self.assertEqual(read.call_count,1)
def test_price_read_filters_and_normalizes_before_calculation(self):
rows=pd.DataFrame({'trade_date':[DAY]*3,'ts_code':['SH600001','000001','830001.BJ'],'close':[10,20,999]})
with patch.object(factors,'_price_code_col',return_value='symbol'),patch.object(common.db,'read_mysql',return_value=rows) as read:
result=factors._read_gp_price(DAY,DAY)
self.assertEqual(list(result.ts_code),['600001.SH','000001.SZ']);self.assertIn('REGEXP',read.call_args[0][1])
def test_upside_only_uses_scoped_consensus_and_prices(self):
cons=pd.DataFrame({'ts_code':['SH600001','BJ830001'],'asof_date':[DAY]*2,'target_mid_avg':[12,1000]})
prices=pd.DataFrame({'ts_code':['600001.SH'],'trade_date':[DAY],'close':[10]})
with patch.object(common.db,'read_pg',return_value=cons) as read, patch.object(factors,'_read_gp_price',return_value=prices),patch.object(common.config,'SUBFACTOR_UNIVERSE','market'):
result=factors.build_upside(DAY,DAY)
self.assertEqual(list(result.stock_code),['SH600001']);self.assertAlmostEqual(result.iloc[0].factor_value,.2)
self.assertIn(' ~ ',read.call_args[0][0])
def test_event_filter_precedes_single_document_cap(self):
ev=pd.DataFrame({'ts_code':['BJ830001','SH600001'],'disclosure_date':[DAY]*2,'event_type':['股份回购']*2,'direction':['']*2,'confidence':[1,.8],'doc_id':[1,1],'source_type':['announcement']*2})
with patch.object(common.db,'read_pg',return_value=ev) as read,patch.object(common.config,'SUBFACTOR_UNIVERSE','market'),patch.object(common.config,'EVENT_SOURCE_TYPES',{'announcement'}),patch.object(common.config,'EVENT_MAX_PER_DOC',1),patch.object(common,'trading_days',return_value=[pd.Timestamp(DAY)]):
result=factors.build_event(DAY,DAY)
self.assertEqual(list(result.stock_code),['600001.SH']);self.assertIn(' ~ ',read.call_args[0][0])
def test_transmission_filters_before_max_and_warning(self):
tr=pd.DataFrame({'scan_date':[DAY]*3,'ts_code':['SH600001','600001.SH','BJ830001'],'n_sources':[1,2,999],'moved_ratio':[0,0,0]})
with patch.object(common.db,'read_pg',return_value=tr) as read,patch.object(common,'load_universe',return_value={'600001.SH','830001.BJ'}),patch.object(factors,'_warn_upstream_truncation') as warn:
result=factors.build_transmission(DAY,DAY)
self.assertEqual(list(result.stock_code),['600001.SH']);self.assertEqual(result.iloc[0].factor_value,2)
self.assertEqual(set(warn.call_args[0][0].ts_code),{'600001.SH'});self.assertIn(' ~ ',read.call_args[0][0])
def test_external_values_do_not_change_score_distribution(self):
def score(add_bad):
codes=['SH600001','SZ000001','SH600002']+(['BJ830001'] if add_bad else [])
upside=frame(codes,[.1,.3,.8]+([1e12] if add_bad else []))
heat=frame(codes,[.2,.4,.9]+([-1e12] if add_bad else []))
transmission=frame(codes,[1,2,4]+([1e12] if add_bad else []))
with patch.object(factors,'_days_of',return_value=[DAY]),patch.object(factors,'_track_set',return_value=None),patch.object(factors,'_risk_set',return_value=set()),patch.object(factors,'build_upside',return_value=upside),patch.object(factors,'_heat_day_for',return_value=DAY),patch.object(factors,'build_heat',return_value=heat),patch.object(factors,'build_transmission',return_value=transmission):
return factors.build_score(DAY,DAY)
pd.testing.assert_frame_equal(score(False),score(True))
def test_risk_members_exclude_bj_before_name_rule(self):
pools=pd.DataFrame({'members':[json.dumps([{'ts_code':'BJ830001','name':'ST范围外'},{'ts_code':'600001','name':'ST有效'}])]})
with patch.object(common.db,'read_pg',return_value=pools),patch.object(common.config,'EXCLUDE_RISK_NAMES',True):
self.assertEqual(factors._risk_set(),{'SH600001'})
def test_calendar_date_query_excludes_out_of_scope(self):
with patch.object(factors,'_price_code_col',return_value='symbol'),patch.object(common.db,'read_mysql',return_value=pd.DataFrame({'trade_date':[DAY]})) as read:
self.assertEqual(len(common.trading_days(DAY,DAY)),1)
self.assertIn('REGEXP',read.call_args[0][1])
def test_write_factor_preserves_historical_bj_and_other_dates(self):
connection=sqlite3.connect(':memory:')
self.addCleanup(connection.close)
connection.create_function('REGEXP',2,lambda regex,value:int(bool(re.fullmatch(regex,str(value)))))
connection.execute('CREATE TABLE t_factor_akg_score(trade_date TEXT,stock_code TEXT,factor_value REAL)')
baseline=[(DAY,'BJ830001',77.),(DAY,'830002.BJ',78.),(DAY,'SH600001',2.),('2026-11-03','SH600001',3.)]
connection.executemany('INSERT INTO t_factor_akg_score VALUES(?,?,?)',baseline);connection.commit()
class Cursor:
def __enter__(self):return self
def __exit__(self,*a):pass
def execute(self,sql,params):return connection.execute(sql.replace('%s','?'),params)
def executemany(self,sql,rows):return connection.executemany(sql.replace('%s','?'),rows)
class Conn:
def cursor(self):return Cursor()
def commit(self):connection.commit()
@contextlib.contextmanager
def conn():yield Conn()
with patch.object(common.db,'factor_conn',conn),patch.object(common,'ensure_table'):
common.write_factor('t_factor_akg_score',frame(['600001','BJ830001','830002.BJ'],[10,1000,1001]))
rows=connection.execute('SELECT * FROM t_factor_akg_score ORDER BY trade_date,stock_code').fetchall()
self.assertEqual(rows,sorted([baseline[0],baseline[1],baseline[3],(DAY,'SH600001',10.)]))
def test_out_of_scope_only_write_never_deletes_or_creates_table(self):
with patch.object(common,'ensure_table') as ensure:
common.write_factor('t_factor_akg_score',frame(BAD))
ensure.assert_not_called()
def test_freeze_nested_members_and_extra_frames_have_no_out_of_scope(self):
members=json.dumps([{'ts_code':'SH600001','name':'有效'},{'ts_code':'830001.BJ','name':'范围外'}, {'ts_code':'600001.SZ'}])
source=pd.DataFrame({'ts_code':['600001.SH','830001.BJ'],'members':[members,members],'details':[{'quiet':['600001','BJ830001']},{}]})
with tempfile.TemporaryDirectory() as root,patch.object(freeze.config,'FROZEN_ROOT',root),patch.object(freeze,'_FMT','csv.gz'),patch.object(freeze,'_EXT','.csv.gz'),patch.object(freeze,'_git_rev',return_value='test'),patch.dict(freeze._SOURCES,{'universe':lambda day:source},clear=True):
manifest=freeze.snapshot(DAY,extra_frames={'factor_akg_score':frame(['600001','BJ830001'])})
result=pd.read_csv(Path(root)/DAY/'universe.csv.gz')
factor=pd.read_csv(Path(root)/DAY/'factor_akg_score.csv.gz')
self.assertEqual(manifest['rows'],{'universe':1,'factor_akg_score':1})
self.assertNotIn('830001',result.to_json());self.assertNotIn('830001',factor.to_json())
self.assertEqual(json.loads(result.iloc[0].members),[{'ts_code':'600001.SH','name':'有效'}])
self.assertEqual(manifest['errors'],{})
def test_industry_edges_remain_and_invalid_member_json_is_error(self):
edge=pd.DataFrame({'src_segment':['产业甲'],'dst_segment':['产业乙'],'mechanism':['[事实] 保持']})
pd.testing.assert_frame_equal(scope.frozen_frame(edge),edge)
with self.assertRaises(ValueError):scope.frozen_frame(pd.DataFrame({'members':['[bad json']}))
def test_all_security_capture_queries_are_scoped(self):
def pg(sql,params=None):
self.assertIn(' ~ ',sql)
return pd.DataFrame({'ts_code':['600001.SH','830001.BJ']})
def mysql(which,sql,params=None):
self.assertIn('REGEXP',sql)
if 'MAX(batch_no)' in sql:return pd.DataFrame({'trade_date':[DAY],'batch_no':[1]})
if which=='heat':return pd.DataFrame({'stock_code':['SH600001','BJ830001'],'trade_date':[DAY]*2,'score':[1,2],'batch_no':[1,1]})
return pd.DataFrame({'code':['600001','830001'],'close':[1,2]})
with patch.object(common.db,'read_pg',side_effect=pg),patch.object(common.db,'read_mysql',side_effect=mysql),patch.object(factors,'_price_code_col',return_value='symbol'):
for fn in (freeze._cap_universe,freeze._cap_transmission,freeze._cap_consensus,freeze._cap_price,freeze._cap_heat):
with self.subTest(fn=fn.__name__):self.assertEqual(len(fn(DAY)),1)
if __name__=='__main__':unittest.main()