akg-factor-bridge/dividend_sources.py

123 lines
6.2 KiB
Python

"""高股息观察取数。只读、单表查询,不调用模型或交易接口。"""
import copy
import datetime as dt
import json
import re
import urllib.parse
import urllib.request
from dividend_watch import BJ, candidates, day, digest, encode, instant, refresh_codes, unique
def rows(cursor):
names = [d[0] for d in cursor.description]
return [dict(zip(names, row)) for row in cursor.fetchall()]
def public_day(value):
if value is None:
return None
raw = str(value)
try:
return dt.datetime.strptime(raw, '%Y%m%d').date() if re.fullmatch(r'\d{8}', raw) else day(raw)
except ValueError:
return None
def public_rows(records, cutoff):
"""原始流量为累计值,保留报告类型,不在这里相加或假定为单季。"""
accepted, quarantined = [], []
for row in sorted(records, key=encode):
dates = [public_day(row[k]) for k in ('ann_date', 'f_ann_date', 'imp_ann_date') if row.get(k)]
if not dates or any(d is None for d in dates) or max(dates) > cutoff:
quarantined.append(row)
else:
accepted.append(row)
return {'records': accepted, 'quarantined': quarantined,
'completeness_verified': False}
def report_read(code, review, previous, base):
"""旧报告不能固定追溯:前后核对最新索引,只保存这次读取的原文副本。"""
if not re.fullmatch(r'\d{6}\.(SH|SZ|BJ)', code):
raise ValueError('代码无效')
identity = digest({k: review.get(k) for k in ('ran_at', 'summary', 'grades')})
old = (previous or {}).get('report_capture') or {}
if old.get('identity') == identity and old.get('status') == 'ok':
return old
ref = (review.get('summary') or {}).get('review_ref')
suffix = '?ref=' + urllib.parse.quote(ref) if ref else ''
url = base.rstrip('/') + f'/company/{code}/review'
def get(path):
with urllib.request.urlopen(path, timeout=15) as response:
return response.read().decode('utf-8')
def check_index():
current = json.loads(get(url + '/summary'))
if instant(current['ran_at']) != instant(review['ran_at']):
raise ValueError('最新报告与冻结索引不同')
try:
if not ref:
check_index()
text = get(url + suffix)
if not ref:
check_index()
return {'status': 'ok', 'identity': identity, 'markdown': text,
'content_hash': digest(text), 'url': url + suffix,
'traceability': 'fixed_reference' if ref else 'captured_current_legacy_report'}
except Exception as exc:
return {'status': 'unavailable', 'identity': identity, 'error_type': type(exc).__name__,
'traceability': 'summary_only', 'url': url + suffix}
def collect(expected, previous=None):
import db
import config
data_day = day(expected)
table = 'gp_stock_factor_pro_' + data_day.strftime('%Y%m')
cutoff = dt.datetime.now(dt.timezone.utc)
out = {'captured_at': cutoff.isoformat(), 'data_date': str(data_day),
'expected_date': str(data_day), 'table': table}
with db._mysql_cm('heat') as connection:
with connection.cursor() as q:
# 不用 MAX 日静默退回陈旧数据,精确查询上一完整交易日。
q.execute(f'SELECT ts_code,symbol,trade_date,close,dv_ratio,dv_ttm,pe_ttm,pb,create_time FROM {table} WHERE trade_date=%s ORDER BY ts_code', (str(data_day),))
out['factors'] = rows(q)
q.execute('SELECT gp_code_two,gp_name,list_status,list_date,industry FROM gp_code_all ORDER BY gp_code_two')
out['names'] = rows(q)
if not out['factors'] or not out['names']:
raise ValueError('预期数据日行情或名称为空')
want = candidates(out)
with db.akg_pg_conn() as connection:
q = connection.execute("SELECT DISTINCT ON (ts_code) ts_code,period,review_version,grades,summary,ran_at,period_fi,period_rf,n_unknown,warnings,report_key,json_key FROM company_review_runs WHERE status='ok' AND ts_code=ANY(%s) AND ran_at<=%s ORDER BY ts_code,ran_at DESC,period DESC,review_version DESC", (want, cutoff))
out['reviews'] = rows(q)
old_reviews = unique((previous or {}).get('reviews', []), 'ts_code')
for review in out['reviews']:
code = review['ts_code']
review['report_capture'] = report_read(code, review, old_reviews.get(code), config.AKG_API_BASE)
review['report_url'] = review['report_capture']['url']
prior_facts = (previous or {}).get('fundamentals') or {}
out['fundamentals'] = {code: copy.deepcopy(prior_facts[code]) for code in want if code in prior_facts}
due = refresh_codes(out, previous)
if due:
marks = ','.join(['%s']*len(due))
lower = (data_day.year-5)*10000 + 101
specs = (
('financials', 'gp_fina_indicator', 'ts_code,ann_date,end_date,eps,dt_eps,roe,roe_dt,roic,netprofit_yoy,ocf_to_profit,debt_to_assets,or_yoy', 'ts_code,end_date,ann_date'),
('cashflow', 'gp_cashflow', 'ts_code,ann_date,f_ann_date,end_date,comp_type,report_type,update_flag,net_profit,n_cashflow_act,n_cashflow_inv_act,c_pay_dist_dpcp_int_exp,free_cashflow', 'ts_code,end_date,ann_date,report_type'),
('dividends', 'gp_dividend', 'ts_code,end_date,ann_date,div_proc,cash_div_tax,stk_div,ex_date,pay_date,imp_ann_date,base_date,base_share', 'ts_code,end_date,ann_date,ex_date,div_proc'),
)
raw = {code: {} for code in due}
with db._mysql_cm('heat') as connection:
with connection.cursor() as q:
for key, name, fields, order in specs:
q.execute(f'SELECT {fields} FROM {name} WHERE ts_code IN ({marks}) AND end_date>=%s ORDER BY {order}', tuple(due)+(lower,))
by_code = {code: [] for code in due}
for row in rows(q):
by_code[row['ts_code']].append(row)
for code in due:
raw[code][key] = public_rows(by_code[code], cutoff.astimezone(BJ).date())
for code in due:
out['fundamentals'][code] = {'checked_at': cutoff.isoformat(), 'sources': raw[code]}
# 统一 Decimal 与日期序列化后再计算指纹,真实取数与离线复演相同。
return json.loads(encode(out))