tradingSystem/app/core/etf_contract/publisher.py

246 lines
12 KiB
Python

"""Redis transport for ETF snapshots and lifecycle events.
The two Redis instances cannot share a transaction. An immutable snapshot is
prepared first; one Lua operation on the event instance commits events,
CAS-protected state and the READY manifest. Only then can a monotonic latest
pointer move and a notification be published. Consumers verify the manifest.
"""
from __future__ import annotations
from datetime import datetime, time, timedelta
import hashlib
import json
from .signals import SHANGHAI, aware_time, lease_until, validate_snapshot
from .timing import LEGACY_POLICY, admission_until, policy_version
from .consumer import OBSERVATION_NAMESPACE, validate_publication_contract, validate_observation_event_metadata
class PublishConflict(RuntimeError):
pass
def _json(value):
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"),
allow_nan=False,
default=lambda x: aware_time(x).isoformat() if isinstance(x, datetime) else _unsupported(x))
def _unsupported(value):
raise TypeError("not JSON serializable: " + type(value).__name__)
def _text(value):
return value.decode() if isinstance(value, bytes) else value
PREPARE_LUA = """-- ETF_PREPARE_V1
local old = redis.call('GET', KEYS[1])
if old and old ~= ARGV[1] then return redis.error_reply('IMMUTABLE_ROUND_CONFLICT') end
if not old then redis.call('SET', KEYS[1], ARGV[1]) end
redis.call('EXPIREAT', KEYS[1], ARGV[2])
return 1
"""
COMMIT_LUA = """-- ETF_COMMIT_V1
local kinds = {'string', 'stream', 'set', 'string'}
for i, key in ipairs(KEYS) do
local kind = redis.call('TYPE', key).ok
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('WRONGTYPE_ETF_KEY') end
end
local done = redis.call('GET', KEYS[4])
if done then return done end
local current = redis.call('GET', KEYS[1])
local actual = 0
if current then actual = tonumber(cjson.decode(current).state_version or 0) end
if ARGV[1] == '1' and actual ~= tonumber(ARGV[2]) then
return redis.error_reply('STATE_CAS_CONFLICT')
end
local events = cjson.decode(ARGV[5])
for _, e in ipairs(events) do
if redis.call('SISMEMBER', KEYS[3], e.key) == 0 then
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[8], '*', 'data', e.data)
redis.call('SADD', KEYS[3], e.key)
end
end
if ARGV[1] == '1' then
redis.call('SET', KEYS[1], ARGV[3])
redis.call('EXPIREAT', KEYS[1], ARGV[6])
end
if #events > 0 then
redis.call('EXPIREAT', KEYS[2], ARGV[7])
redis.call('EXPIREAT', KEYS[3], ARGV[7])
end
redis.call('SET', KEYS[4], ARGV[4])
redis.call('EXPIREAT', KEYS[4], ARGV[7])
return ARGV[4]
"""
LATEST_LUA = """-- ETF_LATEST_V1
local old = redis.call('GET', KEYS[1])
if old then
local v = cjson.decode(old)
if tonumber(v.window_epoch) > tonumber(ARGV[1]) then return 0 end
if tonumber(v.window_epoch) == tonumber(ARGV[1]) and tonumber(v.revision) > tonumber(ARGV[2]) then return 0 end
end
redis.call('SET', KEYS[1], ARGV[3])
redis.call('EXPIREAT', KEYS[1], ARGV[4])
return 1
"""
FAILURE_LUA = """-- ETF_FAILURE_V1
redis.call('SET', KEYS[1], ARGV[1])
redis.call('EXPIREAT', KEYS[1], ARGV[2])
return 1
"""
class RedisPublisher:
def __init__(self, snapshot_client, event_client, namespace="mtf:etf:shadow:v1", maxlen=100000):
if not namespace or namespace.endswith(":" ):
raise ValueError("namespace must be nonempty without a trailing colon")
if maxlen <= 0:
raise ValueError("maxlen must be positive")
self.snapshots = snapshot_client
self.events = event_client
self.namespace = namespace
self.maxlen = int(maxlen)
self.channel = namespace + ":round_ready"
def state_key(self, trade_date):
return f"{self.namespace}:state:{trade_date}"
def load_state(self, trade_date):
raw = self.events.get(self.state_key(trade_date))
return json.loads(_text(raw)) if raw else {}
def _expiry(self, trade_date, days):
close = datetime.combine(datetime.fromisoformat(trade_date).date(), time(15), SHANGHAI)
return int((close + timedelta(days=days)).timestamp())
def publish(self, snapshot, events, new_state, published_at, delivery_at=None):
"""Commit a frozen publication; delivery_at is the real retry clock.
Late retries retain original event timestamps for audit but never move
the live pointer. The runner must pass its current clock as delivery_at.
Omitting it preserves deterministic replay/test behavior.
"""
end = validate_snapshot(snapshot)
publication_metadata = validate_publication_contract(snapshot, self.namespace)
timing_policy = policy_version(snapshot)
now = aware_time(published_at)
if self.namespace == OBSERVATION_NAMESPACE:
if aware_time(snapshot.get("published_at")) != now:
raise ValueError("observation snapshot must retain the frozen publication timestamp")
if type(snapshot.get("session_closed")) is not bool or snapshot["session_closed"] != (end.hour == 15):
raise ValueError("observation session_closed must match the fixed closing window")
delivery = aware_time(delivery_at) if delivery_at is not None else now
if delivery < now:
raise ValueError("delivery_at cannot precede the frozen publication timestamp")
if end > now:
raise ValueError("cannot publish an unfinished future window")
date = snapshot["trade_date"]
revision = snapshot.get("revision", 0)
round_id = snapshot["round_id"]
suffix = f"{date}:{round_id}:r{revision}"
snapshot_key = f"{self.namespace}:snapshot:{suffix}"
manifest_key = f"{self.namespace}:manifest:{suffix}"
snapshot_expiry, stream_expiry = self._expiry(date, 2), self._expiry(date, 7)
canonical = _json(snapshot)
lease = lease_until(end, timing_policy)
admission = admission_until(end, timing_policy)
original_eligible = (snapshot.get("mode", "LIVE") == "LIVE"
and snapshot.get("data_status") == "OK"
and end <= now < admission and not snapshot.get("session_closed", False))
current = original_eligible and end <= delivery < lease
manifest = {
"schema_version": 1, "status": "READY", "round_id": round_id,
"revision": revision, "trade_date": date, "window_end": end.isoformat(),
"window_epoch": int(end.timestamp()), "published_at": now.isoformat(),
"valid_until": lease.isoformat(),
"timing_policy_version": timing_policy, "admission_until": admission.isoformat(),
"session_closed": bool(snapshot.get("session_closed")) or end.hour == 15,
"live_eligible": current,
"mode": snapshot.get("mode", "LIVE"), "snapshot_key": snapshot_key,
"manifest_key": manifest_key, "event_count": len(events),
"snapshot_sha256": hashlib.sha256(canonical.encode()).hexdigest(),
}
if self.namespace == OBSERVATION_NAMESPACE:
manifest.update(publication_mode="observation", **publication_metadata)
# Historical backfills belong in the durable results store. Never
# resurrect Redis data beyond its fixed retention boundary.
if snapshot_expiry <= int(delivery.timestamp()):
return dict(manifest, status="HISTORY_ONLY", cached=False)
if events and snapshot.get("mode", "LIVE") == "REPLAY":
raise ValueError("REPLAY must not publish live lifecycle events")
encoded_events = []
for event in events:
if self.namespace == OBSERVATION_NAMESPACE:
validate_observation_event_metadata(event, snapshot)
event_policy = policy_version(event)
if event.get("round_id") != round_id or event.get("revision") != revision:
raise ValueError("event round/revision does not match snapshot")
if event.get("trade_date") != date or event.get("window_end") != end.isoformat():
raise ValueError("event time partition does not match snapshot")
if aware_time(event["published_at"]) != now:
raise ValueError("retry the exact frozen publication timestamp")
if event.get("event_state") in ("OPEN", "CONTINUE"):
if not original_eligible:
raise ValueError("stale, replay or closed rounds cannot publish active events")
if event_policy != timing_policy:
raise ValueError("active event timing policy must match its snapshot")
if aware_time(event["valid_until"]) > lease_until(end, event_policy):
raise ValueError("event lease cannot extend beyond its window lease or market close")
key = f"{event['event_id']}:{event['event_seq']}:{event['revision']}"
encoded_events.append({"key": key, "data": _json(event)})
write_state = bool(new_state.get("_write_state", False))
if events and not write_state:
raise ValueError("lifecycle events require an atomic state update")
if write_state and (snapshot.get("mode", "LIVE") == "REPLAY" or new_state.get("trade_date") != date):
raise ValueError("state update must belong to this LIVE trading date")
if write_state and policy_version(new_state) != timing_policy:
raise ValueError("state timing policy must match its current snapshot")
expected = int(new_state.get("_expected_version", new_state.get("state_version", 0)))
persisted_state = {k: v for k, v in new_state.items() if not k.startswith("_")}
if write_state and persisted_state.get("state_version") != expected + 1:
raise ValueError("state_version must increment exactly once")
try:
self.snapshots.eval(PREPARE_LUA, 1, snapshot_key, canonical, snapshot_expiry)
committed = self.events.eval(
COMMIT_LUA, 4, self.state_key(date), f"{self.namespace}:events:{date}",
f"{self.namespace}:event_ids:{date}", manifest_key,
"1" if write_state else "0", expected, _json(persisted_state), _json(manifest),
_json(encoded_events), snapshot_expiry, stream_expiry, self.maxlen,
)
manifest = json.loads(_text(committed))
if current:
switched = self.snapshots.eval(
LATEST_LUA, 1, self.namespace + ":latest", int(end.timestamp()), revision,
_json(manifest), snapshot_expiry,
)
if switched:
self.snapshots.publish(self.channel, _json(manifest))
return manifest
except Exception as exc:
if "CONFLICT" in str(exc):
raise PublishConflict(str(exc)) from exc
raise
def publish_failure(self, round_id, trade_date, window_end, error_code, published_at, revision=0,
timing_policy_version=LEGACY_POLICY):
"""Diagnostic status only; never changes a READY manifest or latest."""
end, now = aware_time(window_end), aware_time(published_at)
payload = {
"schema_version": 1, "status": "FAILED", "round_id": round_id,
"trade_date": trade_date, "window_end": end.isoformat(), "revision": revision,
"published_at": now.isoformat(), "error_code": str(error_code),
"timing_policy_version": policy_version({"timing_policy_version": timing_policy_version}),
}
key = f"{self.namespace}:failure:{trade_date}:{round_id}:r{revision}"
self.events.eval(FAILURE_LUA, 1, key, _json(payload), self._expiry(trade_date, 2))
self.snapshots.publish(self.channel, _json(payload))
return payload