270 lines
14 KiB
Python
270 lines
14 KiB
Python
"""Daily observations: ETF net subscription estimates and THS industry flows.
|
||
|
||
The two accounting bases never share a total. All rolling windows use actual
|
||
trading dates and a fixed cohort; missing observations never become zero.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from decimal import Decimal, InvalidOperation
|
||
import hashlib
|
||
import json
|
||
from statistics import median
|
||
|
||
|
||
CONTRACT = "daily-flow-1.0"
|
||
ETF_BASIS = "ETF_SHARE_CHANGE_TIMES_NAV_ESTIMATE"
|
||
INDUSTRY_BASIS = "THS_INDUSTRY_NET_FLOW"
|
||
THRESHOLD = "daily-flow-reference-v0"
|
||
|
||
|
||
def decimal(value):
|
||
if value is None or isinstance(value, bool):
|
||
raise ValueError("MISSING_OR_INVALID_NUMBER")
|
||
try:
|
||
result = Decimal(str(value))
|
||
except InvalidOperation:
|
||
raise ValueError("INVALID_NUMBER") from None
|
||
if not result.is_finite():
|
||
raise ValueError("NONFINITE_NUMBER")
|
||
return result
|
||
|
||
|
||
def canonical(value):
|
||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
|
||
|
||
|
||
def digest(value):
|
||
return hashlib.sha256(canonical(value).encode()).hexdigest()
|
||
|
||
|
||
def iso_day(value):
|
||
from datetime import date
|
||
text = str(value)[:10]
|
||
if len(text) == 8 and text.isdigit():
|
||
text = f"{text[:4]}-{text[4:6]}-{text[6:]}"
|
||
return date.fromisoformat(text).isoformat()
|
||
|
||
|
||
def subscription(current, previous, day, previous_day):
|
||
"""Source total_share=10,000 units, total_size=10,000 CNY, NAV=CNY/unit.
|
||
|
||
This estimates net creation/redemption asset value, including in-kind
|
||
baskets. It is NOT actual cash transferred, secondary-market turnover,
|
||
gross subscriptions/redemptions, or cash entering the stock market.
|
||
"""
|
||
result = dict(trade_date=day, net_flow_yuan=None, previous_size_yuan=None,
|
||
flow_rate=None, status="MISSING_SOURCE", corporate_action_status="UNVERIFIED")
|
||
if current is None or previous is None:
|
||
return result
|
||
if (current.get("ts_code") != previous.get("ts_code")
|
||
or iso_day(current["trade_date"]) != day
|
||
or iso_day(previous["trade_date"]) != previous_day):
|
||
raise ValueError("SHARE_DATE_OR_IDENTITY_MISMATCH")
|
||
try:
|
||
shares, old_shares = decimal(current["total_share"]), decimal(previous["total_share"])
|
||
nav, old_nav = decimal(current["nav"]), decimal(previous["nav"])
|
||
size, old_size = decimal(current["total_size"]), decimal(previous["total_size"])
|
||
if min(shares, old_shares, nav, old_nav, size, old_size) <= 0:
|
||
return dict(result, status="INVALID_SOURCE")
|
||
except (ValueError, KeyError):
|
||
return dict(result, status="INVALID_SOURCE")
|
||
# NAV is rounded by the provider. This is an identity/units sanity bound,
|
||
# not an assertion that the reported size is exactly NAV * rounded shares.
|
||
if any(abs(s * n - a) / a > Decimal("0.001")
|
||
for s, n, a in ((shares, nav, size), (old_shares, old_nav, old_size))):
|
||
return dict(result, status="NAV_SIZE_INCONSISTENT")
|
||
share_ratio, nav_ratio = shares / old_shares, nav / old_nav
|
||
if (abs(share_ratio - 1) > Decimal("0.2") and abs(nav_ratio - 1) > Decimal("0.2")
|
||
and abs(share_ratio * nav_ratio - 1) < Decimal("0.15")):
|
||
return dict(result, status="POSSIBLE_SHARE_UNIT_EVENT", corporate_action_status="REVIEW_REQUIRED")
|
||
flow = (shares - old_shares) * Decimal(10000) * nav
|
||
previous_size = old_size * Decimal(10000)
|
||
return dict(result, status="ESTIMATED", net_flow_yuan=float(flow),
|
||
previous_size_yuan=float(previous_size), flow_rate=float(flow / previous_size),
|
||
share_change_units=float((shares - old_shares) * Decimal(10000)),
|
||
nav_yuan=float(nav), assumption="NO_UNIDENTIFIED_SHARE_UNIT_EVENT")
|
||
|
||
|
||
def aggregate(records, expected_codes, day):
|
||
expected = tuple(sorted(set(expected_codes)))
|
||
selected = [records.get(code) for code in expected]
|
||
valid = [r for r in selected if r and r.get("net_flow_yuan") is not None]
|
||
total = sum((decimal(r["net_flow_yuan"]) for r in valid), Decimal(0))
|
||
base = sum((decimal(r["previous_size_yuan"]) for r in valid), Decimal(0))
|
||
complete = bool(expected) and len(valid) == len(expected)
|
||
return dict(trade_date=day, status="ESTIMATED" if complete else "INCOMPLETE_COHORT",
|
||
net_flow_yuan=float(total) if complete else None,
|
||
observed_partial_net_flow_yuan=float(total) if valid else None,
|
||
previous_size_yuan=float(base) if complete else None,
|
||
flow_rate=float(total / base) if complete and base else None,
|
||
member_count=len(expected), observed_member_count=len(valid),
|
||
missing_codes=[c for c, r in zip(expected, selected) if not r or r.get("net_flow_yuan") is None])
|
||
|
||
|
||
def reference(value, history):
|
||
values = [decimal(v) for v in history if v is not None][-252:]
|
||
result = dict(n=len(values), percentile=None, percentile252=None,
|
||
tail="UNAVAILABLE" if value is None else "INSUFFICIENT_REFERENCE",
|
||
threshold_version=THRESHOLD, calibrated=False,
|
||
percentile_semantics="MIDRANK_DESCRIPTIVE_NOT_PROBABILITY")
|
||
if value is None or not values:
|
||
return result
|
||
x = decimal(value)
|
||
less, equal = sum(v < x for v in values), sum(v == x for v in values)
|
||
rank = 100 * (less + equal / 2) / len(values)
|
||
result.update(percentile=rank, percentile252=rank if len(values) == 252 else None)
|
||
if len(values) < 60:
|
||
return result
|
||
# Use both ends of a tie's rank interval so a constant history cannot be
|
||
# labelled extreme merely because weak ECDF maps ties to 100%.
|
||
lower, upper = 100 * less / len(values), 100 * (less + equal) / len(values)
|
||
result["tail"] = "NONE"
|
||
if x > 0 and lower >= 95:
|
||
result["tail"] = "INFLOW_TOP_1PCT" if len(values) == 252 and lower >= 99 else "INFLOW_TOP_5PCT"
|
||
elif x < 0 and upper <= 5:
|
||
result["tail"] = "OUTFLOW_BOTTOM_1PCT" if len(values) == 252 and upper <= 1 else "OUTFLOW_BOTTOM_5PCT"
|
||
return result
|
||
|
||
|
||
def _window(series, end, length, basis):
|
||
if end + 1 < length:
|
||
return None, None
|
||
rows = series[end + 1 - length:end + 1]
|
||
if any(r.get("net_flow_yuan") is None for r in rows):
|
||
return None, None
|
||
value = sum((decimal(r["net_flow_yuan"]) for r in rows), Decimal(0))
|
||
base = rows[0].get("previous_size_yuan")
|
||
rate = float(value / decimal(base)) if basis == ETF_BASIS and base is not None and decimal(base) > 0 else None
|
||
return float(value), rate
|
||
|
||
|
||
def summarize(series, basis):
|
||
if not series:
|
||
raise ValueError("EMPTY_DAILY_SERIES")
|
||
dates = [iso_day(r["trade_date"]) for r in series]
|
||
if dates != sorted(set(dates)):
|
||
raise ValueError("DATES_MUST_BE_UNIQUE_AND_ORDERED")
|
||
last = series[-1]
|
||
value = last.get("net_flow_yuan")
|
||
direction = "UNKNOWN" if value is None else "INFLOW" if value > 0 else "OUTFLOW" if value < 0 else "FLAT"
|
||
field = "flow_rate" if basis == ETF_BASIS else "net_flow_yuan"
|
||
daily_ref = reference(last.get(field), [r.get(field) for r in series[:-1]])
|
||
result = dict(latest=last, direction=direction, reference=daily_ref, periods={},
|
||
reference_basis="NET_FLOW_DIVIDED_BY_PREVIOUS_SIZE" if basis == ETF_BASIS else "OWN_SECTOR_NET_FLOW_AMOUNT",
|
||
daily_series=series[-60:], model_status="uncalibrated", calibration_status="PROVISIONAL")
|
||
for length in (5, 20, 60):
|
||
amount, rate = _window(series, len(series)-1, length, basis)
|
||
history = [_window(series, i, length, basis)[1 if basis == ETF_BASIS else 0]
|
||
for i in range(length-1, len(series)-1)]
|
||
valid_days = sum(r.get("net_flow_yuan") is not None for r in series[-length:])
|
||
result["periods"][str(length)] = dict(net_flow_yuan=amount, flow_rate=rate,
|
||
expected_days=length, valid_days=valid_days,
|
||
complete=amount is not None, reference=reference(rate if basis == ETF_BASIS else amount, history))
|
||
current5, _ = _window(series, len(series)-1, 5, basis)
|
||
previous5, _ = _window(series, len(series)-6, 5, basis) if len(series) >= 10 else (None, None)
|
||
result["change_in_5d_average_yuan_per_day"] = ((current5-previous5)/5
|
||
if current5 is not None and previous5 is not None else None)
|
||
streak = 0
|
||
if direction in ("INFLOW", "OUTFLOW"):
|
||
for row in reversed(series):
|
||
flow = row.get("net_flow_yuan")
|
||
if flow is None or not ((flow > 0 and direction == "INFLOW") or (flow < 0 and direction == "OUTFLOW")):
|
||
break
|
||
streak += 1
|
||
result["same_direction_trading_days"] = streak
|
||
result["streak_is_lower_bound"] = streak == len(series) and bool(streak)
|
||
label = "ETF净申购估算" if basis == ETF_BASIS else "行业资金净额(同花顺口径)"
|
||
if value is None:
|
||
headline = f"{dates[-1]}:{label}待补数据;不判定流入、流出或极端程度。"
|
||
else:
|
||
headline = f"{dates[-1]}:{label}{value/100000000:+.2f}亿元"
|
||
if basis == ETF_BASIS and last.get("flow_rate") is not None:
|
||
headline += f",占前日规模{last['flow_rate']*100:+.2f}%"
|
||
if daily_ref["percentile"] is not None:
|
||
headline += f",历史分位{daily_ref['percentile']:.1f}%(n={daily_ref['n']})"
|
||
tail_labels = {"INFLOW_TOP_5PCT":"流入前5%档", "OUTFLOW_BOTTOM_5PCT":"流出后5%档",
|
||
"INFLOW_TOP_1PCT":"流入前1%档", "OUTFLOW_BOTTOM_1PCT":"流出后1%档",
|
||
"INSUFFICIENT_REFERENCE":"样本不足,暂不判极端", "NONE":"未达到尾部档位"}
|
||
headline += ";"+tail_labels.get(daily_ref["tail"],"历史参照不足")+";未校准。"
|
||
result["conclusion"] = {"text":headline,"direction":direction,"extreme_code":daily_ref["tail"],
|
||
"is_calibrated":False}
|
||
return result
|
||
|
||
|
||
def build_report(*, dates, share_days, industry_rows, cohort, group_names=None):
|
||
"""dates is a verified trading calendar, including previous boundary day.
|
||
|
||
cohort maps ETF to tracking index. Every historical aggregate is recomputed
|
||
using exactly this cohort; archived market totals from other pools are not
|
||
concatenated. Industry values are already normalized to CNY by the adapter.
|
||
"""
|
||
dates = [iso_day(day) for day in dates]
|
||
if len(dates) < 2 or dates != sorted(set(dates)) or not cohort:
|
||
raise ValueError("VERIFIED_DATES_AND_NONEMPTY_COHORT_REQUIRED")
|
||
share_index = {}
|
||
for day in dates:
|
||
indexed = {}
|
||
for row in share_days.get(day, []):
|
||
code = row["ts_code"]
|
||
if iso_day(row["trade_date"]) != day or code in indexed:
|
||
raise ValueError("DUPLICATE_OR_WRONG_DATE_SHARE_ROW")
|
||
indexed[code] = row
|
||
share_index[day] = indexed
|
||
series = {code: [] for code in cohort}
|
||
groups = {key: sorted(code for code, group in cohort.items() if group == key) for key in sorted(set(cohort.values()))}
|
||
group_series = {key: [] for key in groups}
|
||
market = []
|
||
for previous, day in zip(dates, dates[1:]):
|
||
rows = {code: subscription(share_index[day].get(code), share_index[previous].get(code), day, previous) for code in cohort}
|
||
for code, row in rows.items():
|
||
series[code].append(row)
|
||
market.append(aggregate(rows, cohort, day))
|
||
for group, codes in groups.items():
|
||
group_series[group].append(aggregate(rows, codes, day))
|
||
etfs = [{"subject_type":"ETF", "subject_id":code, "tracking_index":cohort[code],
|
||
**summarize(rows, ETF_BASIS)} for code, rows in sorted(series.items())]
|
||
# Detailed ETF daily history is kept in durable source facts. The public
|
||
# snapshot contains their current/rolling metrics; charts use group series.
|
||
for item in etfs:
|
||
item.pop("daily_series")
|
||
indexes = [{"subject_type":"INDEX", "subject_id":key, "name":(group_names or {}).get(key, key),
|
||
**summarize(rows, ETF_BASIS)} for key, rows in sorted(group_series.items())]
|
||
ind_index, names = {}, {}
|
||
for row in industry_rows:
|
||
day, code = iso_day(row["trade_date"]), str(row["industry_code"])
|
||
if day not in dates[1:]:
|
||
continue
|
||
if (code, day) in ind_index:
|
||
raise ValueError("DUPLICATE_INDUSTRY_ROW")
|
||
value = row.get("net_flow_yuan")
|
||
value = float(decimal(value)) if value is not None else None
|
||
ind_index[code, day] = dict(trade_date=day, net_flow_yuan=value,
|
||
status="REPORTED" if value is not None else "MISSING_SOURCE")
|
||
names[code] = row.get("industry_name", code)
|
||
industries = [{"subject_type":"INDUSTRY", "subject_id":code, "name":names[code],
|
||
**summarize([ind_index.get((code, day), dict(trade_date=day, net_flow_yuan=None,
|
||
status="MISSING_SOURCE")) for day in dates[1:]], INDUSTRY_BASIS)} for code in sorted(names)]
|
||
for subjects in (indexes, industries):
|
||
valid = [r for r in subjects if r["latest"]["net_flow_yuan"] is not None]
|
||
for rank, row in enumerate(sorted(valid, key=lambda r:-r["latest"]["net_flow_yuan"]), 1):
|
||
row["rank_by_net_flow_desc"] = rank
|
||
row["rank_population"] = len(valid)
|
||
return dict(schema_version=1, contract_version=CONTRACT, trade_date=dates[-1],
|
||
publication_mode="observation", purpose="MARKET_OBSERVATION", trade_instruction=False,
|
||
model_status="uncalibrated", calibration_status="PROVISIONAL", threshold_version=THRESHOLD,
|
||
accounting_bases_combined=False, cohort_basis="AS_OF_REPORT_FIXED_COHORT",
|
||
historical_membership_reconstructed=False, cohort_sha256=digest(cohort),
|
||
historical_knowledge_basis="FINALIZED_SOURCE_AS_LOADED_NOT_INTRADAY_REPLAY",
|
||
expected_trading_dates=dates[1:],
|
||
etf={"flow_basis":ETF_BASIS, "amount_unit":"CNY", "cohort":cohort,
|
||
"corporate_action_status":"UNVERIFIED_EXCEPT_SUSPECT_EVENTS_EXCLUDED",
|
||
"interpretation":"ESTIMATED_NET_CREATION_REDEMPTION_ASSET_VALUE_NOT_CASH_OR_STOCK_MARKET_NET_INFLOW",
|
||
"market":{"subject_type":"MARKET", "subject_id":"BROAD_ETF", **summarize(market, ETF_BASIS)},
|
||
"indexes":indexes, "etfs":etfs},
|
||
industry={"flow_basis":INDUSTRY_BASIS, "amount_unit":"CNY", "taxonomy":"THS_INDUSTRY",
|
||
"interpretation":"PROVIDER_REPORTED_ORDER_FLOW_NOT_ETF_SUBSCRIPTION",
|
||
"total":None, "sectors":industries},
|
||
limitations=["Reference tails are provisional descriptive observations, not calibrated probabilities.",
|
||
"ETF creation/redemption can be in kind; estimated flows are not literal cash movement.",
|
||
"Sector differences do not establish that the same investors moved money between sectors."])
|