2026-10-21 16:17:46 +08:00
""" 高股息观察取数。只读、单表查询,不调用模型或交易接口。 """
import copy
import datetime as dt
import json
import re
import urllib . parse
import urllib . request
from dividend_watch import BJ , candidates , day , digest , encode , instant , refresh_codes , unique
def rows ( cursor ) :
names = [ d [ 0 ] for d in cursor . description ]
return [ dict ( zip ( names , row ) ) for row in cursor . fetchall ( ) ]
def public_day ( value ) :
if value is None :
return None
raw = str ( value )
try :
return dt . datetime . strptime ( raw , ' % Y % m %d ' ) . date ( ) if re . fullmatch ( r ' \ d {8} ' , raw ) else day ( raw )
except ValueError :
return None
def public_rows ( records , cutoff ) :
""" 原始流量为累计值,保留报告类型,不在这里相加或假定为单季。 """
accepted , quarantined = [ ] , [ ]
for row in sorted ( records , key = encode ) :
dates = [ public_day ( row [ k ] ) for k in ( ' ann_date ' , ' f_ann_date ' , ' imp_ann_date ' ) if row . get ( k ) ]
if not dates or any ( d is None for d in dates ) or max ( dates ) > cutoff :
quarantined . append ( row )
else :
accepted . append ( row )
return { ' records ' : accepted , ' quarantined ' : quarantined ,
' completeness_verified ' : False }
def report_read ( code , review , previous , base ) :
""" 旧报告不能固定追溯:前后核对最新索引,只保存这次读取的原文副本。 """
if not re . fullmatch ( r ' \ d {6} \ .(SH|SZ|BJ) ' , code ) :
raise ValueError ( ' 代码无效 ' )
identity = digest ( { k : review . get ( k ) for k in ( ' ran_at ' , ' summary ' , ' grades ' ) } )
old = ( previous or { } ) . get ( ' report_capture ' ) or { }
if old . get ( ' identity ' ) == identity and old . get ( ' status ' ) == ' ok ' :
return old
ref = ( review . get ( ' summary ' ) or { } ) . get ( ' review_ref ' )
suffix = ' ?ref= ' + urllib . parse . quote ( ref ) if ref else ' '
url = base . rstrip ( ' / ' ) + f ' /company/ { code } /review '
def get ( path ) :
with urllib . request . urlopen ( path , timeout = 15 ) as response :
return response . read ( ) . decode ( ' utf-8 ' )
def check_index ( ) :
current = json . loads ( get ( url + ' /summary ' ) )
if instant ( current [ ' ran_at ' ] ) != instant ( review [ ' ran_at ' ] ) :
raise ValueError ( ' 最新报告与冻结索引不同 ' )
try :
if not ref :
check_index ( )
text = get ( url + suffix )
if not ref :
check_index ( )
return { ' status ' : ' ok ' , ' identity ' : identity , ' markdown ' : text ,
' content_hash ' : digest ( text ) , ' url ' : url + suffix ,
' traceability ' : ' fixed_reference ' if ref else ' captured_current_legacy_report ' }
except Exception as exc :
return { ' status ' : ' unavailable ' , ' identity ' : identity , ' error_type ' : type ( exc ) . __name__ ,
' traceability ' : ' summary_only ' , ' url ' : url + suffix }
def collect ( expected , previous = None ) :
import db
import config
data_day = day ( expected )
table = ' gp_stock_factor_pro_ ' + data_day . strftime ( ' % Y % m ' )
cutoff = dt . datetime . now ( dt . timezone . utc )
out = { ' captured_at ' : cutoff . isoformat ( ) , ' data_date ' : str ( data_day ) ,
' expected_date ' : str ( data_day ) , ' table ' : table }
with db . _mysql_cm ( ' heat ' ) as connection :
with connection . cursor ( ) as q :
# 不用 MAX 日静默退回陈旧数据,精确查询上一完整交易日。
q . execute ( f ' SELECT ts_code,symbol,trade_date,close,dv_ratio,dv_ttm,pe_ttm,pb,create_time FROM { table } WHERE trade_date=%s ORDER BY ts_code ' , ( str ( data_day ) , ) )
out [ ' factors ' ] = rows ( q )
q . execute ( ' SELECT gp_code_two,gp_name,list_status,list_date,industry FROM gp_code_all ORDER BY gp_code_two ' )
out [ ' names ' ] = rows ( q )
if not out [ ' factors ' ] or not out [ ' names ' ] :
raise ValueError ( ' 预期数据日行情或名称为空 ' )
want = candidates ( out )
2026-10-21 17:17:13 +08:00
from dividend_research_feed import collect as collect_research
out [ ' dividend_research ' ] = collect_research ( want , cutoff . isoformat ( ) , config . AKG_API_BASE )
2026-10-21 16:17:46 +08:00
with db . akg_pg_conn ( ) as connection :
q = connection . execute ( " SELECT DISTINCT ON (ts_code) ts_code,period,review_version,grades,summary,ran_at,period_fi,period_rf,n_unknown,warnings,report_key,json_key FROM company_review_runs WHERE status= ' ok ' AND ts_code=ANY( %s ) AND ran_at<= %s ORDER BY ts_code,ran_at DESC,period DESC,review_version DESC " , ( want , cutoff ) )
out [ ' reviews ' ] = rows ( q )
old_reviews = unique ( ( previous or { } ) . get ( ' reviews ' , [ ] ) , ' ts_code ' )
for review in out [ ' reviews ' ] :
code = review [ ' ts_code ' ]
review [ ' report_capture ' ] = report_read ( code , review , old_reviews . get ( code ) , config . AKG_API_BASE )
review [ ' report_url ' ] = review [ ' report_capture ' ] [ ' url ' ]
prior_facts = ( previous or { } ) . get ( ' fundamentals ' ) or { }
out [ ' fundamentals ' ] = { code : copy . deepcopy ( prior_facts [ code ] ) for code in want if code in prior_facts }
due = refresh_codes ( out , previous )
if due :
marks = ' , ' . join ( [ ' %s ' ] * len ( due ) )
lower = ( data_day . year - 5 ) * 10000 + 101
specs = (
( ' financials ' , ' gp_fina_indicator ' , ' ts_code,ann_date,end_date,eps,dt_eps,roe,roe_dt,roic,netprofit_yoy,ocf_to_profit,debt_to_assets,or_yoy ' , ' ts_code,end_date,ann_date ' ) ,
( ' cashflow ' , ' gp_cashflow ' , ' ts_code,ann_date,f_ann_date,end_date,comp_type,report_type,update_flag,net_profit,n_cashflow_act,n_cashflow_inv_act,c_pay_dist_dpcp_int_exp,free_cashflow ' , ' ts_code,end_date,ann_date,report_type ' ) ,
( ' dividends ' , ' gp_dividend ' , ' ts_code,end_date,ann_date,div_proc,cash_div_tax,stk_div,ex_date,pay_date,imp_ann_date,base_date,base_share ' , ' ts_code,end_date,ann_date,ex_date,div_proc ' ) ,
)
raw = { code : { } for code in due }
with db . _mysql_cm ( ' heat ' ) as connection :
with connection . cursor ( ) as q :
for key , name , fields , order in specs :
q . execute ( f ' SELECT { fields } FROM { name } WHERE ts_code IN ( { marks } ) AND end_date>=%s ORDER BY { order } ' , tuple ( due ) + ( lower , ) )
by_code = { code : [ ] for code in due }
for row in rows ( q ) :
by_code [ row [ ' ts_code ' ] ] . append ( row )
for code in due :
raw [ code ] [ key ] = public_rows ( by_code [ code ] , cutoff . astimezone ( BJ ) . date ( ) )
for code in due :
out [ ' fundamentals ' ] [ code ] = { ' checked_at ' : cutoff . isoformat ( ) , ' sources ' : raw [ code ] }
# 统一 Decimal 与日期序列化后再计算指纹,真实取数与离线复演相同。
return json . loads ( encode ( out ) )