"""高股息独立研究;本地 python3 -m unittest test_dividend_watch。 桥机容器:python dividend_watch.py scan;只读上游,只写 data/dividend_watch。 日检:python dividend_watch.py health。历史复演须使用 --input,不回填当前入口。 """ from __future__ import annotations import argparse import collections import datetime as dt import fcntl import hashlib import html import json import math import os from pathlib import Path import re import tempfile import unicodedata from zoneinfo import ZoneInfo BJ = ZoneInfo('Asia/Shanghai') VERSION = '2' DEFAULT_ROOT = 'data/dividend_watch' LABELS = {'priority': '质地较好,优先核验', 'mixed': '观点有分歧', 'risk': '已有明确风险', 'missing_review': '待补个股评析', 'identity_unknown': '名称或上市状态待核验', 'excluded': '风险名称或非上市,已排除', 'below': '滚动股息不足5%', 'yield_unknown': '滚动股息缺失'} def encode(value): return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':'), allow_nan=False, default=str).encode() def digest(value): return hashlib.sha256(encode(value)).hexdigest() def implementation_fingerprint(): base = Path(__file__).parent return digest({name: hashlib.sha256((base/name).read_bytes()).hexdigest() for name in ('dividend_watch.py', 'dividend_sources.py', 'dividend_research_feed.py')}) def instant(value): result = dt.datetime.fromisoformat(str(value).replace('Z', '+00:00')) if result.tzinfo is None: raise ValueError('截止或来源时间缺少时区') return result def day(value): return dt.date.fromisoformat(str(value)[:10]) def number(value): try: n = float(value) return n if math.isfinite(n) else None except (TypeError, ValueError): return None def risk_name(name): n = re.sub(r'\s+', '', unicodedata.normalize('NFKC', str(name or ''))).upper() while n.startswith(('XD', 'XR', 'DR')): n = n[2:] return n.startswith(('*ST', 'ST', 'SST', 'S*ST', '退市')) or n.endswith('退') def unique(rows, key): result = {} for row in rows: value = row.get(key) if not value or value in result: raise ValueError(f'来源身份缺失或重复:{key}={value}') result[value] = row return result def candidates(source): names = unique(source['names'], 'gp_code_two') factors = unique(source['factors'], 'ts_code') return sorted(code for code, f in factors.items() if (number(f.get('dv_ttm')) or 0) >= 5 and names.get(code, {}).get('gp_name') and names[code].get('list_status') == 'L' and not risk_name(names[code]['gp_name'])) def refresh_codes(source, previous): """月初全查,新进入者当日补;接收新评析不等于重做财务材料。""" local = instant(source['captured_at']).astimezone(BJ) facts = (previous or {}).get('fundamentals') or {} want = candidates(source) due = [c for c in want if c not in facts or instant(facts[c]['checked_at']).astimezone(BJ).strftime('%Y-%m') != local.strftime('%Y-%m')] return due def required_period(on): """按定期披露期限检查材料覆盖,不推断个别公司实际发布日期。""" if on.month >= 11: return f'{on.year}Q3' if on.month >= 9: return f'{on.year}Q2' if on.month >= 5: return f'{on.year}Q1' return f'{on.year-1}Q3' def assemble(source): cutoff = instant(source['captured_at']) data_day = day(source['data_date']) if data_day >= cutoff.astimezone(BJ).date(): raise ValueError('本流程使用上一完整交易日,不接受当日未完成行情') if data_day != day(source['expected_date']): raise ValueError('股息数据日期落后或不匹配') if source['table'] != 'gp_stock_factor_pro_' + data_day.strftime('%Y%m'): raise ValueError('行情日期与月表不符') factors = unique(source['factors'], 'ts_code') names = unique(source['names'], 'gp_code_two') reviews = unique(source.get('reviews', []), 'ts_code') if not factors or not names: raise ValueError('全市场行情或名称源为空') for r in factors.values(): if day(r['trade_date']) != data_day or not re.fullmatch(r'\d{6}\.(SH|SZ|BJ)', r['ts_code']): raise ValueError('混入异日行情或非法代码') for r in reviews.values(): if instant(r['ran_at']) > cutoff: raise ValueError('评析晚于冻结截止时间') rows = [] for code, f in sorted(factors.items()): identity = names.get(code, {}) name = identity.get('gp_name') or '' y = number(f.get('dv_ttm')) rv = reviews.get(code, {}) summary = rv.get('summary') or {} if not isinstance(summary, dict): raise ValueError('评析摘要格式异常') reason = [] if risk_name(name) or (identity.get('list_status') and identity['list_status'] != 'L'): status = 'excluded' elif not name or not identity.get('list_status'): status = 'identity_unknown' elif y is None: status = 'yield_unknown' elif y < 5: status = 'below' elif not rv: status = 'missing_review' elif summary.get('overall') == '差' or summary.get('doubt_hard'): status = 'risk' elif summary.get('overall') == '好': status = 'priority' else: status = 'mixed' material_periods = {k: rv.get(k) or summary.get(k) for k in ('period_fi', 'period_rf')} if rv: if not all(material_periods.values()): reason.append('财务材料期未完整记录') elif len(set(material_periods.values())) > 1: reason.append('两路财务材料期不一致,需补齐较旧一路') required = required_period(cutoff.astimezone(BJ).date()) if any(p and p < required for p in material_periods.values()): reason.append(f'财务指标尚未覆盖应披露的{required},新评析不能替代新材料') if (cutoff - instant(rv['ran_at'])).days > 31: reason.append('个股评析超过一个月,需复核') review_local = instant(rv['ran_at']).astimezone(BJ) current = cutoff.astimezone(BJ) if (review_local.year, (review_local.month-1)//3) != (current.year, (current.month-1)//3): reason.append('本季度尚无完整评析更新') if summary.get('numbers_unmatched'): reason.append('评析数字仍有未核验项') facts = (source.get('fundamentals') or {}).get(code) if status in ('priority', 'mixed', 'risk', 'missing_review'): reason.extend(['普通分红与特别分红需按公告区分', '分红记录完整性及支付能力待核验']) if not facts: reason.append('本月财务与分红事实未读取') elif instant(facts['checked_at']).astimezone(BJ).strftime('%Y-%m') != cutoff.astimezone(BJ).strftime('%Y-%m'): reason.append('财务与分红事实已到月度复查期') else: from dividend_sources import public_day required = required_period(cutoff.astimezone(BJ).date()) year, qtr = required.split('Q') required_end = f'{year}-' + {'1': '03-31', '2': '06-30', '3': '09-30'}[qtr] for key, label in (('financials', '财务指标'), ('cashflow', '现金流报表'), ('dividends', '分红实施材料')): entries = facts['sources'][key]['records'] periods = [public_day(r.get('end_date')) for r in entries] periods = [str(p) for p in periods if p] if not periods: reason.append(label + '缺失') elif key != 'dividends' and max(periods) < required_end: reason.append(label + f'仅到{max(periods)},需补{required}') if status == 'missing_review': reason.insert(0, '需补个股多视角评析及反方观点') if status == 'identity_unknown': reason.append('无法确认最新名称与上市状态,暂不纳入研究池') from dividend_research_feed import display as research_display specialized = (source.get('dividend_research') or {}).get(code) or {} if specialized.get('status') == 'ok' and specialized.get('research'): reason = [x for x in reason if x not in ('普通分红与特别分红需按公告区分', '分红记录完整性及支付能力待核验')] reason.extend(specialized['research'].get('unknowns') or []) rows.append({'code': code, 'name': name, 'status': status, 'yield_ttm_pct': y, 'yield_last_year_pct': number(f.get('dv_ratio')), 'close': number(f.get('close')), 'quality': summary.get('overall'), 'groups': summary.get('groups') or {}, 'perspectives': rv.get('grades') or {}, 'perspectives_text': summary.get('grade_line') or '', 'thesis': summary.get('thesis'), 'antithesis': summary.get('antithesis'), 'warnings': rv.get('warnings') or [], 'quality_reasons': summary.get('quality_reasons') or [], 'unknown_n': summary.get('unknown_n'), 'material_periods': material_periods, 'review_at': rv.get('ran_at'), 'review_ref': summary.get('review_ref'), 'review_url': rv.get('report_url'), 'report_capture': rv.get('report_capture'), 'fundamentals': facts, 'needs': reason, 'dividend_research': specialized, 'sustainability': research_display(specialized), 'investment_approved': False}) counts = dict(collections.Counter(r['status'] for r in rows)) high = [r for r in rows if (r['yield_ttm_pct'] or 0) >= 5] screen = {'market_rows': len(rows), 'high_yield': len(high), 'excluded_high_yield': sum(r['status'] == 'excluded' for r in high), 'identity_unknown_high_yield': sum(r['status'] == 'identity_unknown' for r in high), 'research_pool': sum(r['status'] in ('priority','mixed','risk','missing_review') for r in high)} return {'schema_version': 1, 'code_version': VERSION, 'implementation_fingerprint': implementation_fingerprint(), 'screening': screen, 'data_date': source['data_date'], 'cutoff_at': source['captured_at'], 'source_table': source['table'], 'research_only': True, 'counts': counts, 'rows': rows, 'research_requests': [{'code': r['code'], 'name': r['name'], 'needs': r['needs']} for r in rows if r['status'] in ('priority', 'mixed', 'risk', 'missing_review')]} def fact_summary(facts): if not facts: return None from dividend_sources import public_day labels = {'financials': '财务指标', 'cashflow': '现金流报表', 'dividends': '分红记录'} result = {'本次读取': facts['checked_at'], '说明': '记录条数不代表材料完整。分红性质及支付能力仍须核验。'} for key, item in facts['sources'].items(): periods = [public_day(r.get('end_date')) for r in item['records']] periods = [str(p) for p in periods if p] result[labels.get(key, key)] = {'记录数': len(item['records']), '最新材料期': max(periods) if periods else None, '日期不明或晚于截止的隔离条数': len(item['quarantined'])} return result def next_checks(row): """卡片只列优先核验事项,完整材料缺口仍保留在展开详情。""" if row['status'] == 'identity_unknown': return '先确认最新证券名称与上市状态。' parts = [] special = row.get('dividend_research') or {} research = special.get('research') or {} unknowns = research.get('unknowns', []) if special.get('status') == 'ok' else [] if research.get('cashflow_conflicts'): parts.append('核对不同来源的现金流金额差异') if any('gp_dividend' in n and '冲突' in n for n in unknowns): parts.append('核实有冲突的分红记录') if row['status'] == 'missing_review': parts.append('补齐个股评析与反方观点') if any('材料期' in n or re.search(r'需补\d{4}Q', n) or '财务指标尚未覆盖' in n or ('仅到' in n and '未覆盖' in n) for n in row['needs'] + unknowns): parts.append('补齐最新财报与现金流资料') for item in unknowns: if ('仅到' in item and '未覆盖' in item) or '材料期' in item: continue if ('现金' in item and '不一致' in item) or ('gp_dividend' in item and '冲突' in item): continue if item.startswith('文档') and '节选未覆盖' in item: item = '补查原文未覆盖的相关段落' for name, label in (('gp_cashflow', '现金流报表'), ('gp_fina_indicator', '财务指标'), ('gp_dividend', '分红记录'), ('fin_reformulated', '重构报表')): item = item.replace(name, label) item = item.rstrip('。;') if item not in parts: parts.append(item) if special.get('status') != 'ok': parts.append('核实普通分红及其支付能力') if not parts: parts.append('复核专项研究的支持与反方') return ';'.join(parts[:3]) + '。' def report(result): esc = lambda v: html.escape(str(v if v is not None else '未知')) counts = result['counts'] screening = result['screening'] stats = ' · '.join(f'{LABELS[k]} {counts.get(k, 0)}' for k in ('priority', 'mixed', 'risk', 'missing_review')) content = [] for r in sorted(result['rows'], key=lambda r: (list(LABELS).index(r['status']), r['code'])): if r['status'] in ('below', 'yield_unknown', 'excluded'): continue if r['status'] == 'identity_unknown' and (r['yield_ttm_pct'] or 0) < 5: continue perspectives = r['perspectives_text'] or '视角资料待补齐' detail = esc(json.dumps({'各视角': r['perspectives'], '材料期': r['material_periods'], '完整核查事项': r['needs'], '分红专项研究': r.get('dividend_research'), '来源': r['review_url'], '财务与分红事实': fact_summary(r['fundamentals']), '原报告留存': {k: v for k, v in (r['report_capture'] or {}).items() if k != 'markdown'}}, ensure_ascii=False, indent=2)) content.append(f'

{esc(r["name"] or "名称待核")} ' f'{esc(r["code"])}{r["yield_ttm_pct"]:.2f}%

' f'

{esc(LABELS[r["status"]])} · 分红研究:{esc(r["sustainability"])}

' f'

依据 {esc(r["thesis"] or "尚缺个股评析")}

' f'

反方 {esc(r["antithesis"] or "尚未形成反方研究,不能视为无风险")}

' f'

待查 {esc(next_checks(r))}

' f'
各方观点、材料与来源

{esc(perspectives)}

{detail}
') return '高股息研究观察' + '''''' + f'

高股息研究观察

行情 {esc(result["data_date"])} · 检查 {esc(instant(result["cutoff_at"]).astimezone(BJ).strftime("%Y-%m-%d %H:%M"))} 北京时间

高股息 {screening["high_yield"]} 只 · 风险名称排除 {screening["excluded_high_yield"]} 只 · 身份待核 {screening["identity_unknown_high_yield"]} 只 · 进入研究 {screening["research_pool"]} 只

{esc(stats)}

' + '

滚动股息不低于5%。以下是研究线索,持续分红尚需公告和支付能力证据。

' + ''.join(content) + '''
''' def atomic(path, data): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) fd, tmp = tempfile.mkstemp(dir=path.parent, prefix='.tmp-') try: with os.fdopen(fd, 'wb') as f: f.write(data); f.flush(); os.fsync(f.fileno()) os.replace(tmp, path) finally: if os.path.exists(tmp): os.unlink(tmp) def read_latest(root): p = Path(root)/'latest.json' if not p.exists(): return None info = json.loads(p.read_text()) if not re.fullmatch('[0-9a-f]{64}', info['id']): raise ValueError('研究入口引用无效') raw = (Path(root)/'versions'/info['id']/'bundle.json').read_bytes() if hashlib.sha256(raw).hexdigest() != info['sha256']: raise ValueError('研究归档校验失败') return json.loads(raw) def publish(root, source): root = Path(root); root.mkdir(parents=True, exist_ok=True) source = json.loads(encode(source)) for key, field in (('factors', 'ts_code'), ('names', 'gp_code_two'), ('reviews', 'ts_code')): source[key] = sorted(source.get(key, []), key=lambda row: str(row.get(field) or '')) result = assemble(source) # 采集时刻仅是观察时间。去掉它实现同日同输入重复运行幂等。 effective = {k: v for k,v in source.items() if k != 'captured_at'} identity = digest({'source': effective, 'code_version': VERSION, 'implementation_fingerprint': result['implementation_fingerprint']}) with (root/'publish.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX) old = read_latest(root) if old and old['id'] == identity: return old, False if old and day(old['result']['data_date']) > day(result['data_date']): raise ValueError('不允许旧行情替换最新研究') directory = root/'versions'/identity bundle = {'id': identity, 'parent_id': old['id'] if old else None, 'source': source, 'result': result} path = directory/'bundle.json' if path.exists(): # 中断后沿用已完整写下的首份归档,不覆盖其截止和父引用。 bundle = json.loads(path.read_text()) if bundle['id'] != identity: raise ValueError('研究归档身份不符') else: atomic(path, encode(bundle)) atomic(directory/'report.html', report(bundle['result']).encode()) atomic(root/'latest.json', encode({'id': identity, 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()})) return bundle, True def health(root, expected): try: bundle = read_latest(root) attempt_path = Path(root)/'last_check.json' attempt = json.loads(attempt_path.read_text()) if attempt_path.exists() else {} ok = bool(bundle and day(bundle['result']['data_date']) == day(expected) and attempt.get('ok') and attempt.get('id') == bundle['id']) return {'ok': ok, 'expected_date': str(expected), 'data_date': bundle['result']['data_date'] if bundle else None, 'counts': bundle['result']['counts'] if bundle else {}, 'last_check': attempt} except Exception as exc: return {'ok': False, 'error_type': type(exc).__name__} def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('command', choices=('scan', 'health')) p.add_argument('--input', help='离线冻结输入,复演必须指定独立 --out') p.add_argument('--out', default=DEFAULT_ROOT) p.add_argument('--date', help='日检预期数据日;扫描时只能等于真实上一交易日') a = p.parse_args() if a.input: if a.out == DEFAULT_ROOT or a.command != 'scan': p.error('离线复演只允许 scan 且须指定独立 --out') source = json.loads(Path(a.input).read_text()) expected = source['expected_date'] else: from build_readiness import expected_data_day expected = expected_data_day().isoformat() if a.date and a.command == 'scan' and a.date != expected: p.error('自动观察不能回填历史日期') if a.command == 'health': status = health(a.out, a.date or expected) print(json.dumps(status, ensure_ascii=False)) return 0 if status['ok'] else 1 root = Path(a.out); root.mkdir(parents=True, exist_ok=True) with (root/'scan.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) try: if not a.input: from dividend_sources import collect prior = read_latest(root) source = collect(expected, prior['source'] if prior else None) bundle, changed = publish(root, source) atomic(root/'last_check.json', encode({'ok': True, 'at': dt.datetime.now(BJ).isoformat(), 'id': bundle['id']})) print(json.dumps({'changed': changed, 'id': bundle['id'], 'counts': bundle['result']['counts']}, ensure_ascii=False)) return 0 except Exception as exc: atomic(root/'last_check.json', encode({'ok': False, 'at': dt.datetime.now(BJ).isoformat(), 'error_type': type(exc).__name__})) print('研究检查失败,保留原归档:' + type(exc).__name__) return 1 if __name__ == '__main__': raise SystemExit(main())