246 lines
12 KiB
Python
246 lines
12 KiB
Python
"""Redis transport for ETF snapshots and lifecycle events.
|
|
|
|
The two Redis instances cannot share a transaction. An immutable snapshot is
|
|
prepared first; one Lua operation on the event instance commits events,
|
|
CAS-protected state and the READY manifest. Only then can a monotonic latest
|
|
pointer move and a notification be published. Consumers verify the manifest.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, time, timedelta
|
|
import hashlib
|
|
import json
|
|
|
|
from .signals import SHANGHAI, aware_time, lease_until, validate_snapshot
|
|
from .timing import LEGACY_POLICY, admission_until, policy_version
|
|
from .consumer import OBSERVATION_NAMESPACE, validate_publication_contract, validate_observation_event_metadata
|
|
|
|
|
|
class PublishConflict(RuntimeError):
|
|
pass
|
|
|
|
|
|
def _json(value):
|
|
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"),
|
|
allow_nan=False,
|
|
default=lambda x: aware_time(x).isoformat() if isinstance(x, datetime) else _unsupported(x))
|
|
|
|
|
|
def _unsupported(value):
|
|
raise TypeError("not JSON serializable: " + type(value).__name__)
|
|
|
|
|
|
def _text(value):
|
|
return value.decode() if isinstance(value, bytes) else value
|
|
|
|
|
|
PREPARE_LUA = """-- ETF_PREPARE_V1
|
|
local old = redis.call('GET', KEYS[1])
|
|
if old and old ~= ARGV[1] then return redis.error_reply('IMMUTABLE_ROUND_CONFLICT') end
|
|
if not old then redis.call('SET', KEYS[1], ARGV[1]) end
|
|
redis.call('EXPIREAT', KEYS[1], ARGV[2])
|
|
return 1
|
|
"""
|
|
|
|
|
|
COMMIT_LUA = """-- ETF_COMMIT_V1
|
|
local kinds = {'string', 'stream', 'set', 'string'}
|
|
for i, key in ipairs(KEYS) do
|
|
local kind = redis.call('TYPE', key).ok
|
|
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('WRONGTYPE_ETF_KEY') end
|
|
end
|
|
local done = redis.call('GET', KEYS[4])
|
|
if done then return done end
|
|
local current = redis.call('GET', KEYS[1])
|
|
local actual = 0
|
|
if current then actual = tonumber(cjson.decode(current).state_version or 0) end
|
|
if ARGV[1] == '1' and actual ~= tonumber(ARGV[2]) then
|
|
return redis.error_reply('STATE_CAS_CONFLICT')
|
|
end
|
|
local events = cjson.decode(ARGV[5])
|
|
for _, e in ipairs(events) do
|
|
if redis.call('SISMEMBER', KEYS[3], e.key) == 0 then
|
|
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[8], '*', 'data', e.data)
|
|
redis.call('SADD', KEYS[3], e.key)
|
|
end
|
|
end
|
|
if ARGV[1] == '1' then
|
|
redis.call('SET', KEYS[1], ARGV[3])
|
|
redis.call('EXPIREAT', KEYS[1], ARGV[6])
|
|
end
|
|
if #events > 0 then
|
|
redis.call('EXPIREAT', KEYS[2], ARGV[7])
|
|
redis.call('EXPIREAT', KEYS[3], ARGV[7])
|
|
end
|
|
redis.call('SET', KEYS[4], ARGV[4])
|
|
redis.call('EXPIREAT', KEYS[4], ARGV[7])
|
|
return ARGV[4]
|
|
"""
|
|
|
|
|
|
LATEST_LUA = """-- ETF_LATEST_V1
|
|
local old = redis.call('GET', KEYS[1])
|
|
if old then
|
|
local v = cjson.decode(old)
|
|
if tonumber(v.window_epoch) > tonumber(ARGV[1]) then return 0 end
|
|
if tonumber(v.window_epoch) == tonumber(ARGV[1]) and tonumber(v.revision) > tonumber(ARGV[2]) then return 0 end
|
|
end
|
|
redis.call('SET', KEYS[1], ARGV[3])
|
|
redis.call('EXPIREAT', KEYS[1], ARGV[4])
|
|
return 1
|
|
"""
|
|
|
|
|
|
FAILURE_LUA = """-- ETF_FAILURE_V1
|
|
redis.call('SET', KEYS[1], ARGV[1])
|
|
redis.call('EXPIREAT', KEYS[1], ARGV[2])
|
|
return 1
|
|
"""
|
|
|
|
|
|
class RedisPublisher:
|
|
def __init__(self, snapshot_client, event_client, namespace="mtf:etf:shadow:v1", maxlen=100000):
|
|
if not namespace or namespace.endswith(":" ):
|
|
raise ValueError("namespace must be nonempty without a trailing colon")
|
|
if maxlen <= 0:
|
|
raise ValueError("maxlen must be positive")
|
|
self.snapshots = snapshot_client
|
|
self.events = event_client
|
|
self.namespace = namespace
|
|
self.maxlen = int(maxlen)
|
|
self.channel = namespace + ":round_ready"
|
|
|
|
def state_key(self, trade_date):
|
|
return f"{self.namespace}:state:{trade_date}"
|
|
|
|
def load_state(self, trade_date):
|
|
raw = self.events.get(self.state_key(trade_date))
|
|
return json.loads(_text(raw)) if raw else {}
|
|
|
|
def _expiry(self, trade_date, days):
|
|
close = datetime.combine(datetime.fromisoformat(trade_date).date(), time(15), SHANGHAI)
|
|
return int((close + timedelta(days=days)).timestamp())
|
|
|
|
def publish(self, snapshot, events, new_state, published_at, delivery_at=None):
|
|
"""Commit a frozen publication; delivery_at is the real retry clock.
|
|
|
|
Late retries retain original event timestamps for audit but never move
|
|
the live pointer. The runner must pass its current clock as delivery_at.
|
|
Omitting it preserves deterministic replay/test behavior.
|
|
"""
|
|
end = validate_snapshot(snapshot)
|
|
publication_metadata = validate_publication_contract(snapshot, self.namespace)
|
|
timing_policy = policy_version(snapshot)
|
|
now = aware_time(published_at)
|
|
if self.namespace == OBSERVATION_NAMESPACE:
|
|
if aware_time(snapshot.get("published_at")) != now:
|
|
raise ValueError("observation snapshot must retain the frozen publication timestamp")
|
|
if type(snapshot.get("session_closed")) is not bool or snapshot["session_closed"] != (end.hour == 15):
|
|
raise ValueError("observation session_closed must match the fixed closing window")
|
|
delivery = aware_time(delivery_at) if delivery_at is not None else now
|
|
if delivery < now:
|
|
raise ValueError("delivery_at cannot precede the frozen publication timestamp")
|
|
if end > now:
|
|
raise ValueError("cannot publish an unfinished future window")
|
|
date = snapshot["trade_date"]
|
|
revision = snapshot.get("revision", 0)
|
|
round_id = snapshot["round_id"]
|
|
suffix = f"{date}:{round_id}:r{revision}"
|
|
snapshot_key = f"{self.namespace}:snapshot:{suffix}"
|
|
manifest_key = f"{self.namespace}:manifest:{suffix}"
|
|
snapshot_expiry, stream_expiry = self._expiry(date, 2), self._expiry(date, 7)
|
|
canonical = _json(snapshot)
|
|
lease = lease_until(end, timing_policy)
|
|
admission = admission_until(end, timing_policy)
|
|
original_eligible = (snapshot.get("mode", "LIVE") == "LIVE"
|
|
and snapshot.get("data_status") == "OK"
|
|
and end <= now < admission and not snapshot.get("session_closed", False))
|
|
current = original_eligible and end <= delivery < lease
|
|
manifest = {
|
|
"schema_version": 1, "status": "READY", "round_id": round_id,
|
|
"revision": revision, "trade_date": date, "window_end": end.isoformat(),
|
|
"window_epoch": int(end.timestamp()), "published_at": now.isoformat(),
|
|
"valid_until": lease.isoformat(),
|
|
"timing_policy_version": timing_policy, "admission_until": admission.isoformat(),
|
|
"session_closed": bool(snapshot.get("session_closed")) or end.hour == 15,
|
|
"live_eligible": current,
|
|
"mode": snapshot.get("mode", "LIVE"), "snapshot_key": snapshot_key,
|
|
"manifest_key": manifest_key, "event_count": len(events),
|
|
"snapshot_sha256": hashlib.sha256(canonical.encode()).hexdigest(),
|
|
}
|
|
if self.namespace == OBSERVATION_NAMESPACE:
|
|
manifest.update(publication_mode="observation", **publication_metadata)
|
|
# Historical backfills belong in the durable results store. Never
|
|
# resurrect Redis data beyond its fixed retention boundary.
|
|
if snapshot_expiry <= int(delivery.timestamp()):
|
|
return dict(manifest, status="HISTORY_ONLY", cached=False)
|
|
if events and snapshot.get("mode", "LIVE") == "REPLAY":
|
|
raise ValueError("REPLAY must not publish live lifecycle events")
|
|
encoded_events = []
|
|
for event in events:
|
|
if self.namespace == OBSERVATION_NAMESPACE:
|
|
validate_observation_event_metadata(event, snapshot)
|
|
event_policy = policy_version(event)
|
|
if event.get("round_id") != round_id or event.get("revision") != revision:
|
|
raise ValueError("event round/revision does not match snapshot")
|
|
if event.get("trade_date") != date or event.get("window_end") != end.isoformat():
|
|
raise ValueError("event time partition does not match snapshot")
|
|
if aware_time(event["published_at"]) != now:
|
|
raise ValueError("retry the exact frozen publication timestamp")
|
|
if event.get("event_state") in ("OPEN", "CONTINUE"):
|
|
if not original_eligible:
|
|
raise ValueError("stale, replay or closed rounds cannot publish active events")
|
|
if event_policy != timing_policy:
|
|
raise ValueError("active event timing policy must match its snapshot")
|
|
if aware_time(event["valid_until"]) > lease_until(end, event_policy):
|
|
raise ValueError("event lease cannot extend beyond its window lease or market close")
|
|
key = f"{event['event_id']}:{event['event_seq']}:{event['revision']}"
|
|
encoded_events.append({"key": key, "data": _json(event)})
|
|
write_state = bool(new_state.get("_write_state", False))
|
|
if events and not write_state:
|
|
raise ValueError("lifecycle events require an atomic state update")
|
|
if write_state and (snapshot.get("mode", "LIVE") == "REPLAY" or new_state.get("trade_date") != date):
|
|
raise ValueError("state update must belong to this LIVE trading date")
|
|
if write_state and policy_version(new_state) != timing_policy:
|
|
raise ValueError("state timing policy must match its current snapshot")
|
|
expected = int(new_state.get("_expected_version", new_state.get("state_version", 0)))
|
|
persisted_state = {k: v for k, v in new_state.items() if not k.startswith("_")}
|
|
if write_state and persisted_state.get("state_version") != expected + 1:
|
|
raise ValueError("state_version must increment exactly once")
|
|
try:
|
|
self.snapshots.eval(PREPARE_LUA, 1, snapshot_key, canonical, snapshot_expiry)
|
|
committed = self.events.eval(
|
|
COMMIT_LUA, 4, self.state_key(date), f"{self.namespace}:events:{date}",
|
|
f"{self.namespace}:event_ids:{date}", manifest_key,
|
|
"1" if write_state else "0", expected, _json(persisted_state), _json(manifest),
|
|
_json(encoded_events), snapshot_expiry, stream_expiry, self.maxlen,
|
|
)
|
|
manifest = json.loads(_text(committed))
|
|
if current:
|
|
switched = self.snapshots.eval(
|
|
LATEST_LUA, 1, self.namespace + ":latest", int(end.timestamp()), revision,
|
|
_json(manifest), snapshot_expiry,
|
|
)
|
|
if switched:
|
|
self.snapshots.publish(self.channel, _json(manifest))
|
|
return manifest
|
|
except Exception as exc:
|
|
if "CONFLICT" in str(exc):
|
|
raise PublishConflict(str(exc)) from exc
|
|
raise
|
|
|
|
def publish_failure(self, round_id, trade_date, window_end, error_code, published_at, revision=0,
|
|
timing_policy_version=LEGACY_POLICY):
|
|
"""Diagnostic status only; never changes a READY manifest or latest."""
|
|
end, now = aware_time(window_end), aware_time(published_at)
|
|
payload = {
|
|
"schema_version": 1, "status": "FAILED", "round_id": round_id,
|
|
"trade_date": trade_date, "window_end": end.isoformat(), "revision": revision,
|
|
"published_at": now.isoformat(), "error_code": str(error_code),
|
|
"timing_policy_version": policy_version({"timing_policy_version": timing_policy_version}),
|
|
}
|
|
key = f"{self.namespace}:failure:{trade_date}:{round_id}:r{revision}"
|
|
self.events.eval(FAILURE_LUA, 1, key, _json(payload), self._expiry(trade_date, 2))
|
|
self.snapshots.publish(self.channel, _json(payload))
|
|
return payload
|