338 lines
15 KiB
Python
338 lines
15 KiB
Python
"""宏观网格研究的纯计算。无PMS依赖;缺数不跨接,所有特征只看当时。"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import itertools
|
||
import json
|
||
import math
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import numpy as np
|
||
import pandas as pd
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
ROOT = HERE.parents[1]
|
||
CORE = ['000300.SH', '000905.SH', '000852.SH']
|
||
FOUR = CORE + ['399006.SZ']
|
||
NAMES = {'market': '全市场共识', '000300.SH': '沪深300', '000905.SH': '中证500',
|
||
'000852.SH': '中证1000', '399006.SZ': '创业板', '932000.CSI': '中证2000',
|
||
'000688.SH': '科创50', '000001.SH': '上证指数'}
|
||
BASE = ['r5', 'r20', 'sigma', 'qrs', 'qrs_mean', 'breadth', 'hedge']
|
||
STATIC = ['price_sync', 'volume_sync', 'qrs_sync', 'qrs_dispersion']
|
||
FORMATION = ['price_form', 'volume_form', 'qrs_form', 'breadth_change']
|
||
DIRECTION = ['entry_direction', 'entry_bin_direction', 'dwell', 'new_entry']
|
||
EFFICIENCY = ['efficiency', 'signed_efficiency', 'eff_volume', 'eff_sync', 'eff_position']
|
||
VARIANTS = {'baseline': [], 'grid': [], 'static': [], 'formation': ['F'],
|
||
'direction': ['D'], 'efficiency': ['E'], 'full': ['F', 'D', 'E'],
|
||
'without_formation': ['D', 'E'], 'without_direction': ['F', 'E'],
|
||
'without_efficiency': ['F', 'D']}
|
||
GROUPS = {'F': FORMATION, 'D': DIRECTION, 'E': EFFICIENCY}
|
||
|
||
|
||
def no_network(event, args):
|
||
if event in {'socket.connect', 'socket.getaddrinfo', 'socket.sendto', 'socket.sendmsg'}:
|
||
raise RuntimeError('离线研究禁止联网')
|
||
|
||
|
||
def clean(x):
|
||
if isinstance(x, dict):
|
||
return {str(k): clean(v) for k, v in x.items()}
|
||
if isinstance(x, (list, tuple, np.ndarray)):
|
||
return [clean(v) for v in x]
|
||
if isinstance(x, (pd.Timestamp,)):
|
||
return x.isoformat()
|
||
if isinstance(x, (np.integer,)):
|
||
return int(x)
|
||
if isinstance(x, (np.bool_,)):
|
||
return bool(x)
|
||
if isinstance(x, (float, np.floating)):
|
||
return float(x) if np.isfinite(x) else None
|
||
return x
|
||
|
||
|
||
def dump(path, obj):
|
||
Path(path).write_text(json.dumps(clean(obj), ensure_ascii=False, indent=2, allow_nan=False) + '\n')
|
||
|
||
|
||
def sha(path):
|
||
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
||
|
||
|
||
def protocol():
|
||
digest = sha(HERE / 'protocol.json')
|
||
assert digest == (HERE / 'protocol.sha256').read_text().split()[0], '固定协议摘要不符'
|
||
p = json.loads((HERE / 'protocol.json').read_text())
|
||
for path, expected in p['raw_files'].items():
|
||
assert sha(ROOT / path) == expected, '原始缓存摘要不符'
|
||
return p
|
||
|
||
|
||
def load_data(p):
|
||
raw = json.loads((ROOT / list(p['raw_files'])[0]).read_text())['data']
|
||
micro = json.loads((ROOT / list(p['raw_files'])[1]).read_text())['data']
|
||
rows = {k: v['rows'] for k, v in raw['index_daily'].items()}
|
||
rows['932000.CSI'] = micro['daily']['rows']
|
||
data, quality = {}, {}
|
||
for symbol, records in rows.items():
|
||
d = pd.DataFrame(records)
|
||
d.index = pd.to_datetime(d.pop('timestamp')).dt.normalize()
|
||
d = d.sort_index()
|
||
assert not d.index.duplicated().any(), '重复日期'
|
||
assert d.index.max() <= pd.Timestamp(p['data_cutoff'])
|
||
cols = ['open', 'high', 'low', 'close', 'amount', 'volume']
|
||
d = d[cols].apply(pd.to_numeric, errors='coerce')
|
||
bad = d[['open', 'high', 'low', 'close']].isna().any(axis=1)
|
||
bad |= (d[['open', 'high', 'low', 'close']] <= 0).any(axis=1)
|
||
bad |= d.high < d[['open', 'low', 'close']].max(axis=1) - 1e-6
|
||
bad |= d.low > d[['open', 'high', 'close']].min(axis=1) + 1e-6
|
||
quality[symbol] = {'rows': len(d), 'first': str(d.index.min().date()),
|
||
'last': str(d.index.max().date()), 'invalid_ohlc': int(bad.sum()),
|
||
'invalid_amount': int((~np.isfinite(d.amount) | (d.amount <= 0)).sum())}
|
||
d.loc[bad, ['open', 'high', 'low', 'close']] = np.nan
|
||
d.loc[~np.isfinite(d.amount) | (d.amount <= 0), 'amount'] = np.nan
|
||
data[symbol] = d
|
||
calendar = data['000001.SH'].index
|
||
data = {s: d.reindex(calendar) for s, d in data.items()}
|
||
return data, raw['macro_inputs'], quality
|
||
|
||
|
||
def qrs(d):
|
||
# 与既有上游公式一致;连续窗口使真正缺口后的暖机重新开始。
|
||
low, high = d.low, d.high
|
||
cov = low.rolling(18).cov(high, ddof=0)
|
||
lv, hv = low.rolling(18).var(ddof=0), high.rolling(18).var(ddof=0)
|
||
beta = (cov / lv).where((lv > 0) & (hv > 0))
|
||
r2 = ((cov * cov) / (lv * hv)).clip(0, 1).where(beta.notna())
|
||
sd = beta.rolling(600).std(ddof=0)
|
||
z = ((beta - beta.rolling(600).mean()) / sd).mask(sd == 0, 0)
|
||
mean_r2 = r2.rolling(600).mean()
|
||
result = (z * r2 / mean_r2).mask(mean_r2 == 0, 0)
|
||
valid_bars = d.close.notna().astype(int).rolling(618).sum()
|
||
return result.where(valid_bars == 618)
|
||
|
||
|
||
def past_rank(s, window=252):
|
||
# 严格过去分布,平值取中点;窗口内任何缺失都不出值。
|
||
a = s.to_numpy(float)
|
||
out = np.full(len(a), np.nan)
|
||
for t in range(window, len(a)):
|
||
w = a[t-window:t]
|
||
if np.isfinite(a[t]) and np.isfinite(w).all():
|
||
out[t] = ((w < a[t]).sum() + .5 * (w == a[t]).sum()) / window
|
||
return pd.Series(out, index=s.index)
|
||
|
||
|
||
def bins(rank, n=5):
|
||
return np.floor(rank * n).clip(upper=n-1)
|
||
|
||
|
||
def entry_path(price_bin, actual_change):
|
||
out = np.full((len(price_bin), 4), np.nan)
|
||
last = None
|
||
direction = bin_direction = np.nan
|
||
b, r = price_bin.to_numpy(float), actual_change.to_numpy(float)
|
||
for t in range(len(b)):
|
||
if not np.isfinite(b[t]) or not np.isfinite(r[t]):
|
||
last = None
|
||
continue
|
||
if t == 0 or not np.isfinite(b[t-1]):
|
||
last = None
|
||
continue
|
||
new = b[t] != b[t-1]
|
||
if new:
|
||
last = t
|
||
direction, bin_direction = np.sign(r[t]), np.sign(b[t] - b[t-1])
|
||
if last is not None:
|
||
out[t] = [direction, bin_direction, min(t-last, 20)/20, int(new)]
|
||
return pd.DataFrame(out, index=price_bin.index, columns=DIRECTION)
|
||
|
||
|
||
def own_features(d):
|
||
f = pd.DataFrame(index=d.index)
|
||
prev = d.close.shift(1)
|
||
tr = pd.concat([d.high-d.low, (d.high-prev).abs(), (d.low-prev).abs()], axis=1).max(axis=1, skipna=False)
|
||
f['anchor'] = d.close.shift(1).rolling(60).mean()
|
||
f['atr'] = tr.shift(1).rolling(20).mean()
|
||
f['price_z'] = (d.close - f.anchor) / f.atr
|
||
f['activity'] = d.amount.rolling(5).mean() / d.amount.shift(1).rolling(60).median()
|
||
f['price_rank'] = past_rank(f.price_z)
|
||
f['volume_rank'] = past_rank(f.activity)
|
||
f['price_bin'], f['volume_bin'] = bins(f.price_rank), bins(f.volume_rank)
|
||
f['cell'] = f.price_bin * 5 + f.volume_bin
|
||
f['r5'] = d.close / d.close.shift(5) - 1
|
||
f['r20'] = d.close / d.close.shift(20) - 1
|
||
f['sigma'] = np.log(d.close / prev).rolling(20).std(ddof=1)
|
||
f['qrs'] = qrs(d)
|
||
travel = d.close.diff().abs().rolling(5).sum()
|
||
f['efficiency'] = ((d.close - d.close.shift(5)).abs() / travel).mask(travel == 0, 0)
|
||
f['signed_efficiency'] = ((d.close - d.close.shift(5)) / travel).mask(travel == 0, 0)
|
||
for j in range(6):
|
||
f[f'edge{j}'] = f.price_z.shift(1).rolling(252).quantile(j/5)
|
||
return pd.concat([f, entry_path(f.price_bin, d.close.diff())], axis=1)
|
||
|
||
|
||
def mean_pair_corr(series, window):
|
||
pairs = [series[a].rolling(window).corr(series[b]) for a, b in itertools.combinations(CORE, 2)]
|
||
return pd.concat(pairs, axis=1).mean(axis=1, skipna=False).clip(-1, 1)
|
||
|
||
|
||
def hedge(calendar, close, raw):
|
||
vals, ages = {}, {}
|
||
for key, col, lag in [('fx', 'bid_close', 1), ('shibor', 'rate_1m', 0)]:
|
||
d = pd.DataFrame(raw[key]['rows'])
|
||
dates = pd.to_datetime(d.trade_date) + pd.Timedelta(days=lag)
|
||
s = pd.Series(d[col].to_numpy(float), index=dates).sort_index()
|
||
assert not s.index.duplicated().any()
|
||
vals[key] = s.reindex(calendar, method='ffill')
|
||
obs = pd.Series(s.index, index=s.index).reindex(calendar, method='ffill')
|
||
ages[key] = (pd.Series(calendar, index=calendar)-obs).dt.days
|
||
adjusted = np.log(close/close.shift(20)) + np.log(vals['fx']/vals['fx'].shift(20)) - .02*vals['shibor'].diff(20)
|
||
z = (adjusted-adjusted.rolling(40).mean())/adjusted.rolling(40).std(ddof=1)*10
|
||
return z.where((ages['fx'] <= 6) & (ages['shibor'] <= 6)).round(4)
|
||
|
||
|
||
def make_features(data, macro):
|
||
own = {s: own_features(d) for s, d in data.items() if s != '000001.SH'}
|
||
calendar = next(iter(data.values())).index
|
||
shared = pd.DataFrame(index=calendar)
|
||
inputs = {'price': {s: np.log(data[s].close/data[s].close.shift(1)) for s in CORE},
|
||
'volume': {s: np.log(data[s].amount).diff() for s in CORE},
|
||
'qrs': {s: own[s].qrs.diff() for s in CORE}}
|
||
for kind, series in inputs.items():
|
||
shared[f'{kind}_sync'] = mean_pair_corr(series, 20)
|
||
shared[f'{kind}_form'] = shared[f'{kind}_sync'] - mean_pair_corr(series, 60).shift(20)
|
||
qs = pd.concat([own[s].qrs for s in FOUR], axis=1)
|
||
r5 = pd.concat([own[s].r5 for s in FOUR], axis=1)
|
||
shared['qrs_mean'] = qs.mean(axis=1, skipna=False)
|
||
shared['qrs_dispersion'] = qs.std(axis=1, ddof=0, skipna=False)
|
||
shared['breadth'] = (r5 > 0).mean(axis=1).where(r5.notna().all(axis=1))
|
||
shared['breadth_change'] = shared.breadth.diff(5)
|
||
shared['hedge'] = hedge(calendar, data['000001.SH'].close, macro)
|
||
market = pd.DataFrame(index=calendar)
|
||
for col in ['price_rank', 'volume_rank']:
|
||
market[col] = pd.concat([own[s][col] for s in CORE], axis=1).median(axis=1, skipna=False)
|
||
for col in ['r5', 'r20', 'sigma', 'qrs', 'efficiency', 'signed_efficiency', 'activity', 'price_z']:
|
||
market[col] = pd.concat([own[s][col] for s in CORE], axis=1).mean(axis=1, skipna=False)
|
||
market['price_bin'], market['volume_bin'] = bins(market.price_rank), bins(market.volume_rank)
|
||
market['cell'] = market.price_bin*5 + market.volume_bin
|
||
market_return = pd.concat([data[s].close.pct_change(fill_method=None) for s in CORE], axis=1).mean(axis=1, skipna=False)
|
||
own['market'] = pd.concat([market, entry_path(market.price_bin, market_return)], axis=1)
|
||
for s, f in own.items():
|
||
f = pd.concat([f, shared], axis=1)
|
||
f['eff_volume'] = f.efficiency * np.log(f.activity)
|
||
f['eff_sync'] = f.efficiency * f.price_sync
|
||
f['eff_position'] = f.efficiency * f.price_rank
|
||
own[s] = f.replace([np.inf, -np.inf], np.nan)
|
||
return own
|
||
|
||
|
||
def label_class(z):
|
||
return pd.Series(np.where(z > .5, 2, np.where(z < -.5, 0, 1)), index=z.index).where(z.notna())
|
||
|
||
|
||
def labels(data, features, horizon=20):
|
||
lab = {}
|
||
calendar = next(iter(data.values())).index
|
||
exits = pd.Series(calendar, index=calendar).shift(-(horizon+1))
|
||
entries = pd.Series(calendar, index=calendar).shift(-1)
|
||
for s in features:
|
||
if s == 'market':
|
||
continue
|
||
d, f = data[s], features[s]
|
||
r = d.open.shift(-(horizon+1))/d.open.shift(-1)-1
|
||
# 持有区间有任何缺失价格不产生可兑现标签。
|
||
future_ok = d[['open', 'high', 'low', 'close']].notna().all(axis=1).astype(int).rolling(horizon+1).sum().shift(-(horizon+1)) == horizon+1
|
||
r = r.where(future_ok)
|
||
scale = f.sigma * np.sqrt(horizon)
|
||
z = r / scale
|
||
lab[s] = pd.DataFrame({'return': r, 'scale': scale, 'z': z, 'class': label_class(z),
|
||
'entry_date': entries, 'exit_date': exits})
|
||
market_r = pd.concat([lab[s]['return'] for s in CORE], axis=1).mean(axis=1, skipna=False)
|
||
scale = pd.concat([lab[s]['scale'] for s in CORE], axis=1).mean(axis=1, skipna=False)
|
||
classes = pd.concat([lab[s]['class'] for s in CORE], axis=1)
|
||
mc = pd.Series(np.where((classes == 2).sum(axis=1) >= 2, 2,
|
||
np.where((classes == 0).sum(axis=1) >= 2, 0, 1)), index=calendar).where(classes.notna().all(axis=1))
|
||
lab['market'] = pd.DataFrame({'return': market_r, 'scale': scale, 'z': market_r/scale,
|
||
'class': mc, 'entry_date': entries, 'exit_date': exits})
|
||
return lab
|
||
|
||
|
||
def design(f, name):
|
||
cols = BASE.copy()
|
||
if name not in ['baseline', 'grid']:
|
||
cols += STATIC
|
||
for group in VARIANTS[name]:
|
||
cols += GROUPS[group]
|
||
parts = [f[cols]]
|
||
if name != 'baseline':
|
||
cells = pd.DataFrame({f'cell_{j}': (f.cell == j).astype(float).where(f.cell.notna()) for j in range(1, 25)}, index=f.index)
|
||
parts.append(cells)
|
||
for col in cols[len(BASE):]:
|
||
parts.append(cells.mul(f[col], axis=0).rename(columns=lambda x: f'{col}:{x}'))
|
||
return pd.concat(parts, axis=1)
|
||
|
||
|
||
def split_fold(f, l, year, all_features=None):
|
||
x = design(f, 'full') if all_features is None else all_features
|
||
valid = x.notna().all(axis=1)
|
||
first_test = f.index[f.index.year == year]
|
||
if not len(first_test):
|
||
return None
|
||
for y in range(year-5, year):
|
||
cy = f.index[f.index.year == y]
|
||
good = f.index[(f.index.year == y) & valid]
|
||
if not len(cy) or len(good) < 200 or good[0] != cy[0]:
|
||
return None
|
||
train = f.index[(f.index.year >= year-5) & (f.index.year < year) & valid &
|
||
l['class'].notna() & (l.exit_date < first_test[0])]
|
||
test = f.index[(f.index.year == year) & valid & l['class'].notna()]
|
||
if len(train) < 900:
|
||
return None
|
||
return train, test
|
||
|
||
|
||
def fit_models(xtrain, ytrain, ztrain, xtest):
|
||
from sklearn.linear_model import LogisticRegression, Ridge
|
||
from sklearn.preprocessing import StandardScaler
|
||
scaler = StandardScaler()
|
||
a = scaler.fit_transform(xtrain)
|
||
b = scaler.transform(xtest)
|
||
model = LogisticRegression(C=.01, solver='lbfgs', max_iter=3000, tol=1e-7)
|
||
model.fit(a, ytrain.astype(int))
|
||
assert (model.n_iter_ < 3000).all(), '分类模型未收敛'
|
||
probabilities = np.zeros((len(b), 3))
|
||
probabilities[:, model.classes_] = model.predict_proba(b)
|
||
reg = Ridge(alpha=100).fit(a, ztrain)
|
||
return probabilities, reg.predict(b), {'iterations': int(model.n_iter_.max()), 'columns': a.shape[1]}
|
||
|
||
|
||
def brier(p, cls):
|
||
return np.square(p - np.eye(3)[np.asarray(cls, int)]).sum(axis=1)
|
||
|
||
|
||
def holm(values):
|
||
a = np.asarray(values, float)
|
||
order = np.argsort(a)
|
||
corrected = np.minimum(1, np.maximum.accumulate(a[order] * np.arange(len(a), 0, -1)))
|
||
out = np.empty_like(a)
|
||
out[order] = corrected
|
||
return out
|
||
|
||
|
||
def block_interval(values, calendar, repetitions=2000, block=60, seed=20261029):
|
||
# 完整日历保留缺失,不能将缺失前后直接拼接成邻接日。
|
||
a = values.reindex(calendar).to_numpy(float)
|
||
observed = np.nanmean(a)
|
||
if np.isfinite(a).sum() < 120:
|
||
return {'mean': observed, 'ci95': [None, None], 'p': 1.0}
|
||
rng = np.random.default_rng(seed)
|
||
offsets = np.arange(block)
|
||
draws = np.empty(repetitions)
|
||
for k in range(repetitions):
|
||
starts = rng.integers(0, len(a), size=math.ceil(len(a)/block))
|
||
idx = ((starts[:, None] + offsets) % len(a)).ravel()[:len(a)]
|
||
draws[k] = np.nanmean(a[idx])
|
||
p = (1 + (np.abs(draws-observed) >= abs(observed)).sum()) / (repetitions+1)
|
||
return {'mean': observed, 'ci95': np.quantile(draws, [.025, .975]), 'p': p}
|