125 lines
6.4 KiB
Python
125 lines
6.4 KiB
Python
"""高股息观察取数。只读、单表查询,不调用模型或交易接口。"""
|
|
import copy
|
|
import datetime as dt
|
|
import json
|
|
import re
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
from dividend_watch import BJ, candidates, day, digest, encode, instant, refresh_codes, unique
|
|
|
|
|
|
def rows(cursor):
|
|
names = [d[0] for d in cursor.description]
|
|
return [dict(zip(names, row)) for row in cursor.fetchall()]
|
|
|
|
|
|
def public_day(value):
|
|
if value is None:
|
|
return None
|
|
raw = str(value)
|
|
try:
|
|
return dt.datetime.strptime(raw, '%Y%m%d').date() if re.fullmatch(r'\d{8}', raw) else day(raw)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def public_rows(records, cutoff):
|
|
"""原始流量为累计值,保留报告类型,不在这里相加或假定为单季。"""
|
|
accepted, quarantined = [], []
|
|
for row in sorted(records, key=encode):
|
|
dates = [public_day(row[k]) for k in ('ann_date', 'f_ann_date', 'imp_ann_date') if row.get(k)]
|
|
if not dates or any(d is None for d in dates) or max(dates) > cutoff:
|
|
quarantined.append(row)
|
|
else:
|
|
accepted.append(row)
|
|
return {'records': accepted, 'quarantined': quarantined,
|
|
'completeness_verified': False}
|
|
|
|
|
|
def report_read(code, review, previous, base):
|
|
"""旧报告不能固定追溯:前后核对最新索引,只保存这次读取的原文副本。"""
|
|
if not re.fullmatch(r'\d{6}\.(SH|SZ|BJ)', code):
|
|
raise ValueError('代码无效')
|
|
identity = digest({k: review.get(k) for k in ('ran_at', 'summary', 'grades')})
|
|
old = (previous or {}).get('report_capture') or {}
|
|
if old.get('identity') == identity and old.get('status') == 'ok':
|
|
return old
|
|
ref = (review.get('summary') or {}).get('review_ref')
|
|
suffix = '?ref=' + urllib.parse.quote(ref) if ref else ''
|
|
url = base.rstrip('/') + f'/company/{code}/review'
|
|
def get(path):
|
|
with urllib.request.urlopen(path, timeout=15) as response:
|
|
return response.read().decode('utf-8')
|
|
def check_index():
|
|
current = json.loads(get(url + '/summary'))
|
|
if instant(current['ran_at']) != instant(review['ran_at']):
|
|
raise ValueError('最新报告与冻结索引不同')
|
|
try:
|
|
if not ref:
|
|
check_index()
|
|
text = get(url + suffix)
|
|
if not ref:
|
|
check_index()
|
|
return {'status': 'ok', 'identity': identity, 'markdown': text,
|
|
'content_hash': digest(text), 'url': url + suffix,
|
|
'traceability': 'fixed_reference' if ref else 'captured_current_legacy_report'}
|
|
except Exception as exc:
|
|
return {'status': 'unavailable', 'identity': identity, 'error_type': type(exc).__name__,
|
|
'traceability': 'summary_only', 'url': url + suffix}
|
|
|
|
|
|
def collect(expected, previous=None):
|
|
import db
|
|
import config
|
|
data_day = day(expected)
|
|
table = 'gp_stock_factor_pro_' + data_day.strftime('%Y%m')
|
|
cutoff = dt.datetime.now(dt.timezone.utc)
|
|
out = {'captured_at': cutoff.isoformat(), 'data_date': str(data_day),
|
|
'expected_date': str(data_day), 'table': table}
|
|
with db._mysql_cm('heat') as connection:
|
|
with connection.cursor() as q:
|
|
# 不用 MAX 日静默退回陈旧数据,精确查询上一完整交易日。
|
|
q.execute(f'SELECT ts_code,symbol,trade_date,close,dv_ratio,dv_ttm,pe_ttm,pb,create_time FROM {table} WHERE trade_date=%s ORDER BY ts_code', (str(data_day),))
|
|
out['factors'] = rows(q)
|
|
q.execute('SELECT gp_code_two,gp_name,list_status,list_date,industry FROM gp_code_all ORDER BY gp_code_two')
|
|
out['names'] = rows(q)
|
|
if not out['factors'] or not out['names']:
|
|
raise ValueError('预期数据日行情或名称为空')
|
|
want = candidates(out)
|
|
from dividend_research_feed import collect as collect_research
|
|
out['dividend_research'] = collect_research(want, cutoff.isoformat(), config.AKG_API_BASE)
|
|
with db.akg_pg_conn() as connection:
|
|
q = connection.execute("SELECT DISTINCT ON (ts_code) ts_code,period,review_version,grades,summary,ran_at,period_fi,period_rf,n_unknown,warnings,report_key,json_key FROM company_review_runs WHERE status='ok' AND ts_code=ANY(%s) AND ran_at<=%s ORDER BY ts_code,ran_at DESC,period DESC,review_version DESC", (want, cutoff))
|
|
out['reviews'] = rows(q)
|
|
old_reviews = unique((previous or {}).get('reviews', []), 'ts_code')
|
|
for review in out['reviews']:
|
|
code = review['ts_code']
|
|
review['report_capture'] = report_read(code, review, old_reviews.get(code), config.AKG_API_BASE)
|
|
review['report_url'] = review['report_capture']['url']
|
|
prior_facts = (previous or {}).get('fundamentals') or {}
|
|
out['fundamentals'] = {code: copy.deepcopy(prior_facts[code]) for code in want if code in prior_facts}
|
|
due = refresh_codes(out, previous)
|
|
if due:
|
|
marks = ','.join(['%s']*len(due))
|
|
lower = (data_day.year-5)*10000 + 101
|
|
specs = (
|
|
('financials', 'gp_fina_indicator', 'ts_code,ann_date,end_date,eps,dt_eps,roe,roe_dt,roic,netprofit_yoy,ocf_to_profit,debt_to_assets,or_yoy', 'ts_code,end_date,ann_date'),
|
|
('cashflow', 'gp_cashflow', 'ts_code,ann_date,f_ann_date,end_date,comp_type,report_type,update_flag,net_profit,n_cashflow_act,n_cashflow_inv_act,c_pay_dist_dpcp_int_exp,free_cashflow', 'ts_code,end_date,ann_date,report_type'),
|
|
('dividends', 'gp_dividend', 'ts_code,end_date,ann_date,div_proc,cash_div_tax,stk_div,ex_date,pay_date,imp_ann_date,base_date,base_share', 'ts_code,end_date,ann_date,ex_date,div_proc'),
|
|
)
|
|
raw = {code: {} for code in due}
|
|
with db._mysql_cm('heat') as connection:
|
|
with connection.cursor() as q:
|
|
for key, name, fields, order in specs:
|
|
q.execute(f'SELECT {fields} FROM {name} WHERE ts_code IN ({marks}) AND end_date>=%s ORDER BY {order}', tuple(due)+(lower,))
|
|
by_code = {code: [] for code in due}
|
|
for row in rows(q):
|
|
by_code[row['ts_code']].append(row)
|
|
for code in due:
|
|
raw[code][key] = public_rows(by_code[code], cutoff.astimezone(BJ).date())
|
|
for code in due:
|
|
out['fundamentals'][code] = {'checked_at': cutoff.isoformat(), 'sources': raw[code]}
|
|
# 统一 Decimal 与日期序列化后再计算指纹,真实取数与离线复演相同。
|
|
return json.loads(encode(out))
|