"""高股息观察取数。只读、单表查询,不调用模型或交易接口。""" import copy import datetime as dt import json import re import urllib.parse import urllib.request from dividend_watch import BJ, candidates, day, digest, encode, instant, refresh_codes, unique def rows(cursor): names = [d[0] for d in cursor.description] return [dict(zip(names, row)) for row in cursor.fetchall()] def public_day(value): if value is None: return None raw = str(value) try: return dt.datetime.strptime(raw, '%Y%m%d').date() if re.fullmatch(r'\d{8}', raw) else day(raw) except ValueError: return None def public_rows(records, cutoff): """原始流量为累计值,保留报告类型,不在这里相加或假定为单季。""" accepted, quarantined = [], [] for row in sorted(records, key=encode): dates = [public_day(row[k]) for k in ('ann_date', 'f_ann_date', 'imp_ann_date') if row.get(k)] if not dates or any(d is None for d in dates) or max(dates) > cutoff: quarantined.append(row) else: accepted.append(row) return {'records': accepted, 'quarantined': quarantined, 'completeness_verified': False} def report_read(code, review, previous, base): """旧报告不能固定追溯:前后核对最新索引,只保存这次读取的原文副本。""" if not re.fullmatch(r'\d{6}\.(SH|SZ|BJ)', code): raise ValueError('代码无效') identity = digest({k: review.get(k) for k in ('ran_at', 'summary', 'grades')}) old = (previous or {}).get('report_capture') or {} if old.get('identity') == identity and old.get('status') == 'ok': return old ref = (review.get('summary') or {}).get('review_ref') suffix = '?ref=' + urllib.parse.quote(ref) if ref else '' url = base.rstrip('/') + f'/company/{code}/review' def get(path): with urllib.request.urlopen(path, timeout=15) as response: return response.read().decode('utf-8') def check_index(): current = json.loads(get(url + '/summary')) if instant(current['ran_at']) != instant(review['ran_at']): raise ValueError('最新报告与冻结索引不同') try: if not ref: check_index() text = get(url + suffix) if not ref: check_index() return {'status': 'ok', 'identity': identity, 'markdown': text, 'content_hash': digest(text), 'url': url + suffix, 'traceability': 'fixed_reference' if ref else 'captured_current_legacy_report'} except Exception as exc: return {'status': 'unavailable', 'identity': identity, 'error_type': type(exc).__name__, 'traceability': 'summary_only', 'url': url + suffix} def collect(expected, previous=None): import db import config data_day = day(expected) table = 'gp_stock_factor_pro_' + data_day.strftime('%Y%m') cutoff = dt.datetime.now(dt.timezone.utc) out = {'captured_at': cutoff.isoformat(), 'data_date': str(data_day), 'expected_date': str(data_day), 'table': table} with db._mysql_cm('heat') as connection: with connection.cursor() as q: # 不用 MAX 日静默退回陈旧数据,精确查询上一完整交易日。 q.execute(f'SELECT ts_code,symbol,trade_date,close,dv_ratio,dv_ttm,pe_ttm,pb,create_time FROM {table} WHERE trade_date=%s ORDER BY ts_code', (str(data_day),)) out['factors'] = rows(q) q.execute('SELECT gp_code_two,gp_name,list_status,list_date,industry FROM gp_code_all ORDER BY gp_code_two') out['names'] = rows(q) if not out['factors'] or not out['names']: raise ValueError('预期数据日行情或名称为空') want = candidates(out) from dividend_research_feed import collect as collect_research out['dividend_research'] = collect_research(want, cutoff.isoformat(), config.AKG_API_BASE) with db.akg_pg_conn() as connection: q = connection.execute("SELECT DISTINCT ON (ts_code) ts_code,period,review_version,grades,summary,ran_at,period_fi,period_rf,n_unknown,warnings,report_key,json_key FROM company_review_runs WHERE status='ok' AND ts_code=ANY(%s) AND ran_at<=%s ORDER BY ts_code,ran_at DESC,period DESC,review_version DESC", (want, cutoff)) out['reviews'] = rows(q) old_reviews = unique((previous or {}).get('reviews', []), 'ts_code') for review in out['reviews']: code = review['ts_code'] review['report_capture'] = report_read(code, review, old_reviews.get(code), config.AKG_API_BASE) review['report_url'] = review['report_capture']['url'] prior_facts = (previous or {}).get('fundamentals') or {} out['fundamentals'] = {code: copy.deepcopy(prior_facts[code]) for code in want if code in prior_facts} due = refresh_codes(out, previous) if due: marks = ','.join(['%s']*len(due)) lower = (data_day.year-5)*10000 + 101 specs = ( ('financials', 'gp_fina_indicator', 'ts_code,ann_date,end_date,eps,dt_eps,roe,roe_dt,roic,netprofit_yoy,ocf_to_profit,debt_to_assets,or_yoy', 'ts_code,end_date,ann_date'), ('cashflow', 'gp_cashflow', 'ts_code,ann_date,f_ann_date,end_date,comp_type,report_type,update_flag,net_profit,n_cashflow_act,n_cashflow_inv_act,c_pay_dist_dpcp_int_exp,free_cashflow', 'ts_code,end_date,ann_date,report_type'), ('dividends', 'gp_dividend', 'ts_code,end_date,ann_date,div_proc,cash_div_tax,stk_div,ex_date,pay_date,imp_ann_date,base_date,base_share', 'ts_code,end_date,ann_date,ex_date,div_proc'), ) raw = {code: {} for code in due} with db._mysql_cm('heat') as connection: with connection.cursor() as q: for key, name, fields, order in specs: q.execute(f'SELECT {fields} FROM {name} WHERE ts_code IN ({marks}) AND end_date>=%s ORDER BY {order}', tuple(due)+(lower,)) by_code = {code: [] for code in due} for row in rows(q): by_code[row['ts_code']].append(row) for code in due: raw[code][key] = public_rows(by_code[code], cutoff.astimezone(BJ).date()) for code in due: out['fundamentals'][code] = {'checked_at': cutoff.isoformat(), 'sources': raw[code]} # 统一 Decimal 与日期序列化后再计算指纹,真实取数与离线复演相同。 return json.loads(encode(out))