tradingSystem/app/core/etf_contract/daily_flow.py

270 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Daily observations: ETF net subscription estimates and THS industry flows.
The two accounting bases never share a total. All rolling windows use actual
trading dates and a fixed cohort; missing observations never become zero.
"""
from __future__ import annotations
from decimal import Decimal, InvalidOperation
import hashlib
import json
from statistics import median
CONTRACT = "daily-flow-1.0"
ETF_BASIS = "ETF_SHARE_CHANGE_TIMES_NAV_ESTIMATE"
INDUSTRY_BASIS = "THS_INDUSTRY_NET_FLOW"
THRESHOLD = "daily-flow-reference-v0"
def decimal(value):
if value is None or isinstance(value, bool):
raise ValueError("MISSING_OR_INVALID_NUMBER")
try:
result = Decimal(str(value))
except InvalidOperation:
raise ValueError("INVALID_NUMBER") from None
if not result.is_finite():
raise ValueError("NONFINITE_NUMBER")
return result
def canonical(value):
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
def digest(value):
return hashlib.sha256(canonical(value).encode()).hexdigest()
def iso_day(value):
from datetime import date
text = str(value)[:10]
if len(text) == 8 and text.isdigit():
text = f"{text[:4]}-{text[4:6]}-{text[6:]}"
return date.fromisoformat(text).isoformat()
def subscription(current, previous, day, previous_day):
"""Source total_share=10,000 units, total_size=10,000 CNY, NAV=CNY/unit.
This estimates net creation/redemption asset value, including in-kind
baskets. It is NOT actual cash transferred, secondary-market turnover,
gross subscriptions/redemptions, or cash entering the stock market.
"""
result = dict(trade_date=day, net_flow_yuan=None, previous_size_yuan=None,
flow_rate=None, status="MISSING_SOURCE", corporate_action_status="UNVERIFIED")
if current is None or previous is None:
return result
if (current.get("ts_code") != previous.get("ts_code")
or iso_day(current["trade_date"]) != day
or iso_day(previous["trade_date"]) != previous_day):
raise ValueError("SHARE_DATE_OR_IDENTITY_MISMATCH")
try:
shares, old_shares = decimal(current["total_share"]), decimal(previous["total_share"])
nav, old_nav = decimal(current["nav"]), decimal(previous["nav"])
size, old_size = decimal(current["total_size"]), decimal(previous["total_size"])
if min(shares, old_shares, nav, old_nav, size, old_size) <= 0:
return dict(result, status="INVALID_SOURCE")
except (ValueError, KeyError):
return dict(result, status="INVALID_SOURCE")
# NAV is rounded by the provider. This is an identity/units sanity bound,
# not an assertion that the reported size is exactly NAV * rounded shares.
if any(abs(s * n - a) / a > Decimal("0.001")
for s, n, a in ((shares, nav, size), (old_shares, old_nav, old_size))):
return dict(result, status="NAV_SIZE_INCONSISTENT")
share_ratio, nav_ratio = shares / old_shares, nav / old_nav
if (abs(share_ratio - 1) > Decimal("0.2") and abs(nav_ratio - 1) > Decimal("0.2")
and abs(share_ratio * nav_ratio - 1) < Decimal("0.15")):
return dict(result, status="POSSIBLE_SHARE_UNIT_EVENT", corporate_action_status="REVIEW_REQUIRED")
flow = (shares - old_shares) * Decimal(10000) * nav
previous_size = old_size * Decimal(10000)
return dict(result, status="ESTIMATED", net_flow_yuan=float(flow),
previous_size_yuan=float(previous_size), flow_rate=float(flow / previous_size),
share_change_units=float((shares - old_shares) * Decimal(10000)),
nav_yuan=float(nav), assumption="NO_UNIDENTIFIED_SHARE_UNIT_EVENT")
def aggregate(records, expected_codes, day):
expected = tuple(sorted(set(expected_codes)))
selected = [records.get(code) for code in expected]
valid = [r for r in selected if r and r.get("net_flow_yuan") is not None]
total = sum((decimal(r["net_flow_yuan"]) for r in valid), Decimal(0))
base = sum((decimal(r["previous_size_yuan"]) for r in valid), Decimal(0))
complete = bool(expected) and len(valid) == len(expected)
return dict(trade_date=day, status="ESTIMATED" if complete else "INCOMPLETE_COHORT",
net_flow_yuan=float(total) if complete else None,
observed_partial_net_flow_yuan=float(total) if valid else None,
previous_size_yuan=float(base) if complete else None,
flow_rate=float(total / base) if complete and base else None,
member_count=len(expected), observed_member_count=len(valid),
missing_codes=[c for c, r in zip(expected, selected) if not r or r.get("net_flow_yuan") is None])
def reference(value, history):
values = [decimal(v) for v in history if v is not None][-252:]
result = dict(n=len(values), percentile=None, percentile252=None,
tail="UNAVAILABLE" if value is None else "INSUFFICIENT_REFERENCE",
threshold_version=THRESHOLD, calibrated=False,
percentile_semantics="MIDRANK_DESCRIPTIVE_NOT_PROBABILITY")
if value is None or not values:
return result
x = decimal(value)
less, equal = sum(v < x for v in values), sum(v == x for v in values)
rank = 100 * (less + equal / 2) / len(values)
result.update(percentile=rank, percentile252=rank if len(values) == 252 else None)
if len(values) < 60:
return result
# Use both ends of a tie's rank interval so a constant history cannot be
# labelled extreme merely because weak ECDF maps ties to 100%.
lower, upper = 100 * less / len(values), 100 * (less + equal) / len(values)
result["tail"] = "NONE"
if x > 0 and lower >= 95:
result["tail"] = "INFLOW_TOP_1PCT" if len(values) == 252 and lower >= 99 else "INFLOW_TOP_5PCT"
elif x < 0 and upper <= 5:
result["tail"] = "OUTFLOW_BOTTOM_1PCT" if len(values) == 252 and upper <= 1 else "OUTFLOW_BOTTOM_5PCT"
return result
def _window(series, end, length, basis):
if end + 1 < length:
return None, None
rows = series[end + 1 - length:end + 1]
if any(r.get("net_flow_yuan") is None for r in rows):
return None, None
value = sum((decimal(r["net_flow_yuan"]) for r in rows), Decimal(0))
base = rows[0].get("previous_size_yuan")
rate = float(value / decimal(base)) if basis == ETF_BASIS and base is not None and decimal(base) > 0 else None
return float(value), rate
def summarize(series, basis):
if not series:
raise ValueError("EMPTY_DAILY_SERIES")
dates = [iso_day(r["trade_date"]) for r in series]
if dates != sorted(set(dates)):
raise ValueError("DATES_MUST_BE_UNIQUE_AND_ORDERED")
last = series[-1]
value = last.get("net_flow_yuan")
direction = "UNKNOWN" if value is None else "INFLOW" if value > 0 else "OUTFLOW" if value < 0 else "FLAT"
field = "flow_rate" if basis == ETF_BASIS else "net_flow_yuan"
daily_ref = reference(last.get(field), [r.get(field) for r in series[:-1]])
result = dict(latest=last, direction=direction, reference=daily_ref, periods={},
reference_basis="NET_FLOW_DIVIDED_BY_PREVIOUS_SIZE" if basis == ETF_BASIS else "OWN_SECTOR_NET_FLOW_AMOUNT",
daily_series=series[-60:], model_status="uncalibrated", calibration_status="PROVISIONAL")
for length in (5, 20, 60):
amount, rate = _window(series, len(series)-1, length, basis)
history = [_window(series, i, length, basis)[1 if basis == ETF_BASIS else 0]
for i in range(length-1, len(series)-1)]
valid_days = sum(r.get("net_flow_yuan") is not None for r in series[-length:])
result["periods"][str(length)] = dict(net_flow_yuan=amount, flow_rate=rate,
expected_days=length, valid_days=valid_days,
complete=amount is not None, reference=reference(rate if basis == ETF_BASIS else amount, history))
current5, _ = _window(series, len(series)-1, 5, basis)
previous5, _ = _window(series, len(series)-6, 5, basis) if len(series) >= 10 else (None, None)
result["change_in_5d_average_yuan_per_day"] = ((current5-previous5)/5
if current5 is not None and previous5 is not None else None)
streak = 0
if direction in ("INFLOW", "OUTFLOW"):
for row in reversed(series):
flow = row.get("net_flow_yuan")
if flow is None or not ((flow > 0 and direction == "INFLOW") or (flow < 0 and direction == "OUTFLOW")):
break
streak += 1
result["same_direction_trading_days"] = streak
result["streak_is_lower_bound"] = streak == len(series) and bool(streak)
label = "ETF净申购估算" if basis == ETF_BASIS else "行业资金净额(同花顺口径)"
if value is None:
headline = f"{dates[-1]}:{label}待补数据;不判定流入、流出或极端程度。"
else:
headline = f"{dates[-1]}:{label}{value/100000000:+.2f}亿元"
if basis == ETF_BASIS and last.get("flow_rate") is not None:
headline += f",占前日规模{last['flow_rate']*100:+.2f}%"
if daily_ref["percentile"] is not None:
headline += f",历史分位{daily_ref['percentile']:.1f}%(n={daily_ref['n']})"
tail_labels = {"INFLOW_TOP_5PCT":"流入前5%档", "OUTFLOW_BOTTOM_5PCT":"流出后5%档",
"INFLOW_TOP_1PCT":"流入前1%档", "OUTFLOW_BOTTOM_1PCT":"流出后1%档",
"INSUFFICIENT_REFERENCE":"样本不足,暂不判极端", "NONE":"未达到尾部档位"}
headline += ";"+tail_labels.get(daily_ref["tail"],"历史参照不足")+";未校准。"
result["conclusion"] = {"text":headline,"direction":direction,"extreme_code":daily_ref["tail"],
"is_calibrated":False}
return result
def build_report(*, dates, share_days, industry_rows, cohort, group_names=None):
"""dates is a verified trading calendar, including previous boundary day.
cohort maps ETF to tracking index. Every historical aggregate is recomputed
using exactly this cohort; archived market totals from other pools are not
concatenated. Industry values are already normalized to CNY by the adapter.
"""
dates = [iso_day(day) for day in dates]
if len(dates) < 2 or dates != sorted(set(dates)) or not cohort:
raise ValueError("VERIFIED_DATES_AND_NONEMPTY_COHORT_REQUIRED")
share_index = {}
for day in dates:
indexed = {}
for row in share_days.get(day, []):
code = row["ts_code"]
if iso_day(row["trade_date"]) != day or code in indexed:
raise ValueError("DUPLICATE_OR_WRONG_DATE_SHARE_ROW")
indexed[code] = row
share_index[day] = indexed
series = {code: [] for code in cohort}
groups = {key: sorted(code for code, group in cohort.items() if group == key) for key in sorted(set(cohort.values()))}
group_series = {key: [] for key in groups}
market = []
for previous, day in zip(dates, dates[1:]):
rows = {code: subscription(share_index[day].get(code), share_index[previous].get(code), day, previous) for code in cohort}
for code, row in rows.items():
series[code].append(row)
market.append(aggregate(rows, cohort, day))
for group, codes in groups.items():
group_series[group].append(aggregate(rows, codes, day))
etfs = [{"subject_type":"ETF", "subject_id":code, "tracking_index":cohort[code],
**summarize(rows, ETF_BASIS)} for code, rows in sorted(series.items())]
# Detailed ETF daily history is kept in durable source facts. The public
# snapshot contains their current/rolling metrics; charts use group series.
for item in etfs:
item.pop("daily_series")
indexes = [{"subject_type":"INDEX", "subject_id":key, "name":(group_names or {}).get(key, key),
**summarize(rows, ETF_BASIS)} for key, rows in sorted(group_series.items())]
ind_index, names = {}, {}
for row in industry_rows:
day, code = iso_day(row["trade_date"]), str(row["industry_code"])
if day not in dates[1:]:
continue
if (code, day) in ind_index:
raise ValueError("DUPLICATE_INDUSTRY_ROW")
value = row.get("net_flow_yuan")
value = float(decimal(value)) if value is not None else None
ind_index[code, day] = dict(trade_date=day, net_flow_yuan=value,
status="REPORTED" if value is not None else "MISSING_SOURCE")
names[code] = row.get("industry_name", code)
industries = [{"subject_type":"INDUSTRY", "subject_id":code, "name":names[code],
**summarize([ind_index.get((code, day), dict(trade_date=day, net_flow_yuan=None,
status="MISSING_SOURCE")) for day in dates[1:]], INDUSTRY_BASIS)} for code in sorted(names)]
for subjects in (indexes, industries):
valid = [r for r in subjects if r["latest"]["net_flow_yuan"] is not None]
for rank, row in enumerate(sorted(valid, key=lambda r:-r["latest"]["net_flow_yuan"]), 1):
row["rank_by_net_flow_desc"] = rank
row["rank_population"] = len(valid)
return dict(schema_version=1, contract_version=CONTRACT, trade_date=dates[-1],
publication_mode="observation", purpose="MARKET_OBSERVATION", trade_instruction=False,
model_status="uncalibrated", calibration_status="PROVISIONAL", threshold_version=THRESHOLD,
accounting_bases_combined=False, cohort_basis="AS_OF_REPORT_FIXED_COHORT",
historical_membership_reconstructed=False, cohort_sha256=digest(cohort),
historical_knowledge_basis="FINALIZED_SOURCE_AS_LOADED_NOT_INTRADAY_REPLAY",
expected_trading_dates=dates[1:],
etf={"flow_basis":ETF_BASIS, "amount_unit":"CNY", "cohort":cohort,
"corporate_action_status":"UNVERIFIED_EXCEPT_SUSPECT_EVENTS_EXCLUDED",
"interpretation":"ESTIMATED_NET_CREATION_REDEMPTION_ASSET_VALUE_NOT_CASH_OR_STOCK_MARKET_NET_INFLOW",
"market":{"subject_type":"MARKET", "subject_id":"BROAD_ETF", **summarize(market, ETF_BASIS)},
"indexes":indexes, "etfs":etfs},
industry={"flow_basis":INDUSTRY_BASIS, "amount_unit":"CNY", "taxonomy":"THS_INDUSTRY",
"interpretation":"PROVIDER_REPORTED_ORDER_FLOW_NOT_ETF_SUBSCRIPTION",
"total":None, "sectors":industries},
limitations=["Reference tails are provisional descriptive observations, not calibrated probabilities.",
"ETF creation/redemption can be in kind; estimated flows are not literal cash movement.",
"Sector differences do not establish that the same investors moved money between sectors."])