tradingSystem/research/macro_grid_20261029/engine.py

338 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""宏观网格研究的纯计算。无PMS依赖;缺数不跨接,所有特征只看当时。"""
from __future__ import annotations
import hashlib
import itertools
import json
import math
import sys
from pathlib import Path
import numpy as np
import pandas as pd
HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[1]
CORE = ['000300.SH', '000905.SH', '000852.SH']
FOUR = CORE + ['399006.SZ']
NAMES = {'market': '全市场共识', '000300.SH': '沪深300', '000905.SH': '中证500',
'000852.SH': '中证1000', '399006.SZ': '创业板', '932000.CSI': '中证2000',
'000688.SH': '科创50', '000001.SH': '上证指数'}
BASE = ['r5', 'r20', 'sigma', 'qrs', 'qrs_mean', 'breadth', 'hedge']
STATIC = ['price_sync', 'volume_sync', 'qrs_sync', 'qrs_dispersion']
FORMATION = ['price_form', 'volume_form', 'qrs_form', 'breadth_change']
DIRECTION = ['entry_direction', 'entry_bin_direction', 'dwell', 'new_entry']
EFFICIENCY = ['efficiency', 'signed_efficiency', 'eff_volume', 'eff_sync', 'eff_position']
VARIANTS = {'baseline': [], 'grid': [], 'static': [], 'formation': ['F'],
'direction': ['D'], 'efficiency': ['E'], 'full': ['F', 'D', 'E'],
'without_formation': ['D', 'E'], 'without_direction': ['F', 'E'],
'without_efficiency': ['F', 'D']}
GROUPS = {'F': FORMATION, 'D': DIRECTION, 'E': EFFICIENCY}
def no_network(event, args):
if event in {'socket.connect', 'socket.getaddrinfo', 'socket.sendto', 'socket.sendmsg'}:
raise RuntimeError('离线研究禁止联网')
def clean(x):
if isinstance(x, dict):
return {str(k): clean(v) for k, v in x.items()}
if isinstance(x, (list, tuple, np.ndarray)):
return [clean(v) for v in x]
if isinstance(x, (pd.Timestamp,)):
return x.isoformat()
if isinstance(x, (np.integer,)):
return int(x)
if isinstance(x, (np.bool_,)):
return bool(x)
if isinstance(x, (float, np.floating)):
return float(x) if np.isfinite(x) else None
return x
def dump(path, obj):
Path(path).write_text(json.dumps(clean(obj), ensure_ascii=False, indent=2, allow_nan=False) + '\n')
def sha(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def protocol():
digest = sha(HERE / 'protocol.json')
assert digest == (HERE / 'protocol.sha256').read_text().split()[0], '固定协议摘要不符'
p = json.loads((HERE / 'protocol.json').read_text())
for path, expected in p['raw_files'].items():
assert sha(ROOT / path) == expected, '原始缓存摘要不符'
return p
def load_data(p):
raw = json.loads((ROOT / list(p['raw_files'])[0]).read_text())['data']
micro = json.loads((ROOT / list(p['raw_files'])[1]).read_text())['data']
rows = {k: v['rows'] for k, v in raw['index_daily'].items()}
rows['932000.CSI'] = micro['daily']['rows']
data, quality = {}, {}
for symbol, records in rows.items():
d = pd.DataFrame(records)
d.index = pd.to_datetime(d.pop('timestamp')).dt.normalize()
d = d.sort_index()
assert not d.index.duplicated().any(), '重复日期'
assert d.index.max() <= pd.Timestamp(p['data_cutoff'])
cols = ['open', 'high', 'low', 'close', 'amount', 'volume']
d = d[cols].apply(pd.to_numeric, errors='coerce')
bad = d[['open', 'high', 'low', 'close']].isna().any(axis=1)
bad |= (d[['open', 'high', 'low', 'close']] <= 0).any(axis=1)
bad |= d.high < d[['open', 'low', 'close']].max(axis=1) - 1e-6
bad |= d.low > d[['open', 'high', 'close']].min(axis=1) + 1e-6
quality[symbol] = {'rows': len(d), 'first': str(d.index.min().date()),
'last': str(d.index.max().date()), 'invalid_ohlc': int(bad.sum()),
'invalid_amount': int((~np.isfinite(d.amount) | (d.amount <= 0)).sum())}
d.loc[bad, ['open', 'high', 'low', 'close']] = np.nan
d.loc[~np.isfinite(d.amount) | (d.amount <= 0), 'amount'] = np.nan
data[symbol] = d
calendar = data['000001.SH'].index
data = {s: d.reindex(calendar) for s, d in data.items()}
return data, raw['macro_inputs'], quality
def qrs(d):
# 与既有上游公式一致;连续窗口使真正缺口后的暖机重新开始。
low, high = d.low, d.high
cov = low.rolling(18).cov(high, ddof=0)
lv, hv = low.rolling(18).var(ddof=0), high.rolling(18).var(ddof=0)
beta = (cov / lv).where((lv > 0) & (hv > 0))
r2 = ((cov * cov) / (lv * hv)).clip(0, 1).where(beta.notna())
sd = beta.rolling(600).std(ddof=0)
z = ((beta - beta.rolling(600).mean()) / sd).mask(sd == 0, 0)
mean_r2 = r2.rolling(600).mean()
result = (z * r2 / mean_r2).mask(mean_r2 == 0, 0)
valid_bars = d.close.notna().astype(int).rolling(618).sum()
return result.where(valid_bars == 618)
def past_rank(s, window=252):
# 严格过去分布,平值取中点;窗口内任何缺失都不出值。
a = s.to_numpy(float)
out = np.full(len(a), np.nan)
for t in range(window, len(a)):
w = a[t-window:t]
if np.isfinite(a[t]) and np.isfinite(w).all():
out[t] = ((w < a[t]).sum() + .5 * (w == a[t]).sum()) / window
return pd.Series(out, index=s.index)
def bins(rank, n=5):
return np.floor(rank * n).clip(upper=n-1)
def entry_path(price_bin, actual_change):
out = np.full((len(price_bin), 4), np.nan)
last = None
direction = bin_direction = np.nan
b, r = price_bin.to_numpy(float), actual_change.to_numpy(float)
for t in range(len(b)):
if not np.isfinite(b[t]) or not np.isfinite(r[t]):
last = None
continue
if t == 0 or not np.isfinite(b[t-1]):
last = None
continue
new = b[t] != b[t-1]
if new:
last = t
direction, bin_direction = np.sign(r[t]), np.sign(b[t] - b[t-1])
if last is not None:
out[t] = [direction, bin_direction, min(t-last, 20)/20, int(new)]
return pd.DataFrame(out, index=price_bin.index, columns=DIRECTION)
def own_features(d):
f = pd.DataFrame(index=d.index)
prev = d.close.shift(1)
tr = pd.concat([d.high-d.low, (d.high-prev).abs(), (d.low-prev).abs()], axis=1).max(axis=1, skipna=False)
f['anchor'] = d.close.shift(1).rolling(60).mean()
f['atr'] = tr.shift(1).rolling(20).mean()
f['price_z'] = (d.close - f.anchor) / f.atr
f['activity'] = d.amount.rolling(5).mean() / d.amount.shift(1).rolling(60).median()
f['price_rank'] = past_rank(f.price_z)
f['volume_rank'] = past_rank(f.activity)
f['price_bin'], f['volume_bin'] = bins(f.price_rank), bins(f.volume_rank)
f['cell'] = f.price_bin * 5 + f.volume_bin
f['r5'] = d.close / d.close.shift(5) - 1
f['r20'] = d.close / d.close.shift(20) - 1
f['sigma'] = np.log(d.close / prev).rolling(20).std(ddof=1)
f['qrs'] = qrs(d)
travel = d.close.diff().abs().rolling(5).sum()
f['efficiency'] = ((d.close - d.close.shift(5)).abs() / travel).mask(travel == 0, 0)
f['signed_efficiency'] = ((d.close - d.close.shift(5)) / travel).mask(travel == 0, 0)
for j in range(6):
f[f'edge{j}'] = f.price_z.shift(1).rolling(252).quantile(j/5)
return pd.concat([f, entry_path(f.price_bin, d.close.diff())], axis=1)
def mean_pair_corr(series, window):
pairs = [series[a].rolling(window).corr(series[b]) for a, b in itertools.combinations(CORE, 2)]
return pd.concat(pairs, axis=1).mean(axis=1, skipna=False).clip(-1, 1)
def hedge(calendar, close, raw):
vals, ages = {}, {}
for key, col, lag in [('fx', 'bid_close', 1), ('shibor', 'rate_1m', 0)]:
d = pd.DataFrame(raw[key]['rows'])
dates = pd.to_datetime(d.trade_date) + pd.Timedelta(days=lag)
s = pd.Series(d[col].to_numpy(float), index=dates).sort_index()
assert not s.index.duplicated().any()
vals[key] = s.reindex(calendar, method='ffill')
obs = pd.Series(s.index, index=s.index).reindex(calendar, method='ffill')
ages[key] = (pd.Series(calendar, index=calendar)-obs).dt.days
adjusted = np.log(close/close.shift(20)) + np.log(vals['fx']/vals['fx'].shift(20)) - .02*vals['shibor'].diff(20)
z = (adjusted-adjusted.rolling(40).mean())/adjusted.rolling(40).std(ddof=1)*10
return z.where((ages['fx'] <= 6) & (ages['shibor'] <= 6)).round(4)
def make_features(data, macro):
own = {s: own_features(d) for s, d in data.items() if s != '000001.SH'}
calendar = next(iter(data.values())).index
shared = pd.DataFrame(index=calendar)
inputs = {'price': {s: np.log(data[s].close/data[s].close.shift(1)) for s in CORE},
'volume': {s: np.log(data[s].amount).diff() for s in CORE},
'qrs': {s: own[s].qrs.diff() for s in CORE}}
for kind, series in inputs.items():
shared[f'{kind}_sync'] = mean_pair_corr(series, 20)
shared[f'{kind}_form'] = shared[f'{kind}_sync'] - mean_pair_corr(series, 60).shift(20)
qs = pd.concat([own[s].qrs for s in FOUR], axis=1)
r5 = pd.concat([own[s].r5 for s in FOUR], axis=1)
shared['qrs_mean'] = qs.mean(axis=1, skipna=False)
shared['qrs_dispersion'] = qs.std(axis=1, ddof=0, skipna=False)
shared['breadth'] = (r5 > 0).mean(axis=1).where(r5.notna().all(axis=1))
shared['breadth_change'] = shared.breadth.diff(5)
shared['hedge'] = hedge(calendar, data['000001.SH'].close, macro)
market = pd.DataFrame(index=calendar)
for col in ['price_rank', 'volume_rank']:
market[col] = pd.concat([own[s][col] for s in CORE], axis=1).median(axis=1, skipna=False)
for col in ['r5', 'r20', 'sigma', 'qrs', 'efficiency', 'signed_efficiency', 'activity', 'price_z']:
market[col] = pd.concat([own[s][col] for s in CORE], axis=1).mean(axis=1, skipna=False)
market['price_bin'], market['volume_bin'] = bins(market.price_rank), bins(market.volume_rank)
market['cell'] = market.price_bin*5 + market.volume_bin
market_return = pd.concat([data[s].close.pct_change(fill_method=None) for s in CORE], axis=1).mean(axis=1, skipna=False)
own['market'] = pd.concat([market, entry_path(market.price_bin, market_return)], axis=1)
for s, f in own.items():
f = pd.concat([f, shared], axis=1)
f['eff_volume'] = f.efficiency * np.log(f.activity)
f['eff_sync'] = f.efficiency * f.price_sync
f['eff_position'] = f.efficiency * f.price_rank
own[s] = f.replace([np.inf, -np.inf], np.nan)
return own
def label_class(z):
return pd.Series(np.where(z > .5, 2, np.where(z < -.5, 0, 1)), index=z.index).where(z.notna())
def labels(data, features, horizon=20):
lab = {}
calendar = next(iter(data.values())).index
exits = pd.Series(calendar, index=calendar).shift(-(horizon+1))
entries = pd.Series(calendar, index=calendar).shift(-1)
for s in features:
if s == 'market':
continue
d, f = data[s], features[s]
r = d.open.shift(-(horizon+1))/d.open.shift(-1)-1
# 持有区间有任何缺失价格不产生可兑现标签。
future_ok = d[['open', 'high', 'low', 'close']].notna().all(axis=1).astype(int).rolling(horizon+1).sum().shift(-(horizon+1)) == horizon+1
r = r.where(future_ok)
scale = f.sigma * np.sqrt(horizon)
z = r / scale
lab[s] = pd.DataFrame({'return': r, 'scale': scale, 'z': z, 'class': label_class(z),
'entry_date': entries, 'exit_date': exits})
market_r = pd.concat([lab[s]['return'] for s in CORE], axis=1).mean(axis=1, skipna=False)
scale = pd.concat([lab[s]['scale'] for s in CORE], axis=1).mean(axis=1, skipna=False)
classes = pd.concat([lab[s]['class'] for s in CORE], axis=1)
mc = pd.Series(np.where((classes == 2).sum(axis=1) >= 2, 2,
np.where((classes == 0).sum(axis=1) >= 2, 0, 1)), index=calendar).where(classes.notna().all(axis=1))
lab['market'] = pd.DataFrame({'return': market_r, 'scale': scale, 'z': market_r/scale,
'class': mc, 'entry_date': entries, 'exit_date': exits})
return lab
def design(f, name):
cols = BASE.copy()
if name not in ['baseline', 'grid']:
cols += STATIC
for group in VARIANTS[name]:
cols += GROUPS[group]
parts = [f[cols]]
if name != 'baseline':
cells = pd.DataFrame({f'cell_{j}': (f.cell == j).astype(float).where(f.cell.notna()) for j in range(1, 25)}, index=f.index)
parts.append(cells)
for col in cols[len(BASE):]:
parts.append(cells.mul(f[col], axis=0).rename(columns=lambda x: f'{col}:{x}'))
return pd.concat(parts, axis=1)
def split_fold(f, l, year, all_features=None):
x = design(f, 'full') if all_features is None else all_features
valid = x.notna().all(axis=1)
first_test = f.index[f.index.year == year]
if not len(first_test):
return None
for y in range(year-5, year):
cy = f.index[f.index.year == y]
good = f.index[(f.index.year == y) & valid]
if not len(cy) or len(good) < 200 or good[0] != cy[0]:
return None
train = f.index[(f.index.year >= year-5) & (f.index.year < year) & valid &
l['class'].notna() & (l.exit_date < first_test[0])]
test = f.index[(f.index.year == year) & valid & l['class'].notna()]
if len(train) < 900:
return None
return train, test
def fit_models(xtrain, ytrain, ztrain, xtest):
from sklearn.linear_model import LogisticRegression, Ridge
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
a = scaler.fit_transform(xtrain)
b = scaler.transform(xtest)
model = LogisticRegression(C=.01, solver='lbfgs', max_iter=3000, tol=1e-7)
model.fit(a, ytrain.astype(int))
assert (model.n_iter_ < 3000).all(), '分类模型未收敛'
probabilities = np.zeros((len(b), 3))
probabilities[:, model.classes_] = model.predict_proba(b)
reg = Ridge(alpha=100).fit(a, ztrain)
return probabilities, reg.predict(b), {'iterations': int(model.n_iter_.max()), 'columns': a.shape[1]}
def brier(p, cls):
return np.square(p - np.eye(3)[np.asarray(cls, int)]).sum(axis=1)
def holm(values):
a = np.asarray(values, float)
order = np.argsort(a)
corrected = np.minimum(1, np.maximum.accumulate(a[order] * np.arange(len(a), 0, -1)))
out = np.empty_like(a)
out[order] = corrected
return out
def block_interval(values, calendar, repetitions=2000, block=60, seed=20261029):
# 完整日历保留缺失,不能将缺失前后直接拼接成邻接日。
a = values.reindex(calendar).to_numpy(float)
observed = np.nanmean(a)
if np.isfinite(a).sum() < 120:
return {'mean': observed, 'ci95': [None, None], 'p': 1.0}
rng = np.random.default_rng(seed)
offsets = np.arange(block)
draws = np.empty(repetitions)
for k in range(repetitions):
starts = rng.integers(0, len(a), size=math.ceil(len(a)/block))
idx = ((starts[:, None] + offsets) % len(a)).ravel()[:len(a)]
draws[k] = np.nanmean(a[idx])
p = (1 + (np.abs(draws-observed) >= abs(observed)).sum()) / (repetitions+1)
return {'mean': observed, 'ci95': np.quantile(draws, [.025, .975]), 'p': p}