每日选股计划:区制与评析修订有界串行受理(台账 068)
一、08:45 同刻触发造成区制丢失 XXL 122 的区制追加被 137 五分钟修订占用拒绝,日志 92246 返回 409。两类完整修订改为最多两条等待的串行受理,各自保留任务标识与回调。 二、失败释放与互斥保持 任务结束后原子移交运行槽,前一个失败也继续下一条。普通构建与写池继续忙时拒绝。原 08:45 时刻、五分钟周期、选股排序及所有策略参数保持。 三、离线验证及边界 调度专项 12 例包含两方向并发、失败释放、队列上限和构建隔离。区制测试从旧文件覆盖改为完整版本、晨版不可变和重复追加幂等。七组离线测试通过。候选镜像和线上判收另记。队列使用现有进程内存,重启不冒充完成。
This commit is contained in:
parent
4e846a5a5f
commit
72efc8c5ef
|
|
@ -683,3 +683,12 @@
|
||||||
|
|
||||||
|
|
||||||
- **067 部署补记。** e44f770 本地及候选镜像各 36 组禁网回归通过,10-16 07:33:40 桥重建生效。今晨版本原字节保持,晚生成 10-14 样本已排除,旧格式原件未恢复。07:35 XXL 137 首次自动检查成功,07:35:36 155 提议重核已开启,其他参数保持。08:55 接续实际消费与区制修订,无额外五日观察。证据与不可恢复边界见同日交接 10-16 追加节。
|
- **067 部署补记。** e44f770 本地及候选镜像各 36 组禁网回归通过,10-16 07:33:40 桥重建生效。今晨版本原字节保持,晚生成 10-14 样本已排除,旧格式原件未恢复。07:35 XXL 137 首次自动检查成功,07:35:36 155 提议重核已开启,其他参数保持。08:55 接续实际消费与区制修订,无额外五日观察。证据与不可恢复边界见同日交接 10-16 追加节。
|
||||||
|
|
||||||
|
|
||||||
|
## 068 · 2026-10-16 · 区制与五分钟修订有界串行受理
|
||||||
|
|
||||||
|
- **日期。** 2026-10-16,北京时间。
|
||||||
|
- **改动。** 两类完整修订同刻触发时进入最多两条的等待队列,前一完成或失败均继续下一条,各自结案。普通构建继续忙时拒绝。
|
||||||
|
- **依据。** 08:45 XXL 122 被同刻 137 占用拒绝,日志 92246 返回 409,区制未追加。两个方向、故障释放与队列上限加入回归。
|
||||||
|
- **预期。** 区制与评析按顺序发布完整版本,原晨版和归档不变。原调度时刻、排序、参数与提议规则不变。
|
||||||
|
- **复核日期。** 10-16 本地与镜像禁网测试后部署,只补执行原区制任务;通过正常盘中消费继续判收。
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
"""regime.py 离线单测(无需网络、无需库):未配置 -> UNKNOWN;两种快照形状都能归一;
|
"""regime.py 离线单测(无需网络、无需库):未配置 -> UNKNOWN;两种快照形状都能归一;
|
||||||
弱势日按旋钮判;追加进临时快照并能读回。
|
弱势日按旋钮判;追加进临时快照并能读回。
|
||||||
|
|
||||||
跑法:python3 test_regime.py 或 pytest test_regime.py
|
跑法:python3 test_regime.py;依赖使用项目环境,网络与来源均为替身。
|
||||||
"""
|
"""
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
|
|
@ -78,10 +78,36 @@ def main():
|
||||||
and not os.path.exists(regime.snapshot_path("2026-09-01")))
|
and not os.path.exists(regime.snapshot_path("2026-09-01")))
|
||||||
with open(regime.snapshot_path("2026-09-01"), "w", encoding="utf-8") as f:
|
with open(regime.snapshot_path("2026-09-01"), "w", encoding="utf-8") as f:
|
||||||
json.dump({"date": "2026-09-01", "main": []}, f)
|
json.dump({"date": "2026-09-01", "main": []}, f)
|
||||||
regime.append_to_snapshot("2026-09-01")
|
import datetime as dt
|
||||||
back = regime.read_from_snapshot("2026-09-01")
|
from pathlib import Path
|
||||||
t("追加后能从快照读回 regime 段", back and back["status"] == "OK" and back["weak_count"] == 3)
|
from unittest.mock import patch
|
||||||
t("快照其余内容不丢", json.load(open(regime.snapshot_path("2026-09-01"), encoding="utf-8"))["main"] == [])
|
import plan, plan_refresh, plan_versions as pv
|
||||||
|
now = dt.datetime(2026, 9, 2, 0, 45, tzinfo=dt.timezone.utc)
|
||||||
|
actual_refresh = plan_refresh.refresh
|
||||||
|
def at_fixed_time(*args, **kwargs):
|
||||||
|
return actual_refresh(*args, now=now, **kwargs)
|
||||||
|
path = Path(regime.snapshot_path("2026-09-01"))
|
||||||
|
legacy = path.read_bytes()
|
||||||
|
with patch('plan_refresh.refresh', side_effect=at_fixed_time):
|
||||||
|
regime.append_to_snapshot("2026-09-01")
|
||||||
|
t("旧格式不补造完整晨版,原字节保持", path.read_bytes() == legacy)
|
||||||
|
path.unlink() # 仅删除测试临时夹具,另起完整晨版场景。
|
||||||
|
morning = {'date': '2026-09-01', 'plan_version': 'test', 'main': [], 'observe': []}
|
||||||
|
with pv.writer(d):
|
||||||
|
old = pv.publish(d, morning, {'records': {}, 'market': None, 'regime': None}, {},
|
||||||
|
cutoff_at='2026-09-01T23:10:00+00:00', reason='晨版生成')
|
||||||
|
archive = Path(d)/'versions'/'2026-09-01'/(old['decision']['id']+'.json')
|
||||||
|
before = archive.read_bytes()
|
||||||
|
def assembled(*args, **kwargs):
|
||||||
|
return dict(morning, _full={'main': [], 'observe': []})
|
||||||
|
with patch.object(plan, 'collect', side_effect=assembled), patch.object(plan, 'write_published_md'):
|
||||||
|
regime.append_to_snapshot('2026-09-01')
|
||||||
|
back = regime.read_from_snapshot('2026-09-01')
|
||||||
|
t("追加后完整版本可读区制", back and back['status']=='OK' and back['weak_count']==3)
|
||||||
|
t("晨版固定且旧归档不变", archive.read_bytes()==before and pv.load(d,'2026-09-01',morning=True)==old)
|
||||||
|
t("发布独立完整修订", len(pv.history(d,'2026-09-01'))==2)
|
||||||
|
regime.append_to_snapshot('2026-09-01')
|
||||||
|
t("同区制重复追加不发布空版本", len(pv.history(d,'2026-09-01'))==2)
|
||||||
finally:
|
finally:
|
||||||
urllib.request.urlopen = orig
|
urllib.request.urlopen = orig
|
||||||
print("ALL OK — 环境标签:未配置 / 两种形状 / 弱势日旋钮 / 关闭 / 不可达 / 快照追加 全部通过")
|
print("ALL OK — 环境标签:未配置 / 两种形状 / 弱势日旋钮 / 关闭 / 不可达 / 快照追加 全部通过")
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,7 @@ def _reset(key="test-key"):
|
||||||
config.XXL_CALLBACK_MAX_RETRIES = 0
|
config.XXL_CALLBACK_MAX_RETRIES = 0
|
||||||
xxl._jobs.clear()
|
xxl._jobs.clear()
|
||||||
xxl._running_id = None
|
xxl._running_id = None
|
||||||
|
xxl._pending.clear()
|
||||||
ran, sent = [], []
|
ran, sent = [], []
|
||||||
xxl._run_step = lambda cmd: (ran.append(list(cmd)), (0, "ok"))[1]
|
xxl._run_step = lambda cmd: (ran.append(list(cmd)), (0, "ok"))[1]
|
||||||
xxl._post_callback = lambda url, data: (sent.append((url, data)), "delivered http=200")[1]
|
xxl._post_callback = lambda url, data: (sent.append((url, data)), "delivered http=200")[1]
|
||||||
|
|
@ -185,6 +186,76 @@ def _():
|
||||||
assert any("放弃" in ln for ln in job["tail"]), job["tail"]
|
assert any("放弃" in ln for ln in job["tail"]), job["tail"]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@case("评析和区制同时触发:两方向串行,各自结案一次")
|
||||||
|
def _():
|
||||||
|
import threading
|
||||||
|
for first, second in [("plan-refresh", "regime-append"), ("regime-append", "plan-refresh")]:
|
||||||
|
ran, sent = _reset()
|
||||||
|
release = threading.Event()
|
||||||
|
entered = threading.Event()
|
||||||
|
def slow(cmd):
|
||||||
|
ran.append(cmd[1])
|
||||||
|
if len(ran) == 1:
|
||||||
|
entered.set()
|
||||||
|
assert release.wait(3)
|
||||||
|
return 0, "ok"
|
||||||
|
xxl._run_step = slow
|
||||||
|
a = _trigger(steps=first, callbackUrl="http://xxl/a")
|
||||||
|
assert entered.wait(2)
|
||||||
|
try:
|
||||||
|
b = _trigger(steps=second, callbackUrl="http://xxl/b")
|
||||||
|
assert xxl._jobs[b["task_id"]]["state"] == "queued"
|
||||||
|
assert xxl._jobs[b["task_id"]]["started_at"] is None
|
||||||
|
assert ran == [first]
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
assert _wait_done(a["task_id"])["state"] == "success"
|
||||||
|
assert _wait_done(b["task_id"])["state"] == "success"
|
||||||
|
assert ran == [first, second]
|
||||||
|
assert sorted(x[0] for x in sent) == ["http://xxl/a", "http://xxl/b"]
|
||||||
|
assert not xxl._pending and xxl._running_id is None
|
||||||
|
|
||||||
|
|
||||||
|
@case("修订失败后放行等待项,队列有界且拒绝混入普通构建")
|
||||||
|
def _():
|
||||||
|
import threading
|
||||||
|
ran, sent = _reset()
|
||||||
|
release = threading.Event()
|
||||||
|
entered = threading.Event()
|
||||||
|
def slow(cmd):
|
||||||
|
ran.append(cmd[1])
|
||||||
|
if len(ran) == 1:
|
||||||
|
entered.set()
|
||||||
|
assert release.wait(3)
|
||||||
|
return 1, "source unavailable"
|
||||||
|
return 0, "ok"
|
||||||
|
xxl._run_step = slow
|
||||||
|
a = _trigger(steps="plan-refresh", callbackUrl="http://xxl/a")
|
||||||
|
assert entered.wait(2)
|
||||||
|
try:
|
||||||
|
for step in ["build", "plan-refresh,build"]:
|
||||||
|
try:
|
||||||
|
_trigger(steps=step)
|
||||||
|
raise AssertionError("普通构建不得进入修订队列")
|
||||||
|
except HTTPException as e:
|
||||||
|
assert e.status_code == 409
|
||||||
|
bs = [_trigger(steps="regime-append", callbackUrl=f"http://xxl/b{i}") for i in range(2)]
|
||||||
|
try:
|
||||||
|
_trigger(steps="plan-refresh")
|
||||||
|
raise AssertionError("等待队列不得无限增长")
|
||||||
|
except HTTPException as e:
|
||||||
|
assert e.status_code == 409
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
assert _wait_done(a["task_id"])["state"] == "failed"
|
||||||
|
for b in bs:
|
||||||
|
assert _wait_done(b["task_id"])["state"] == "success"
|
||||||
|
assert ran == ["plan-refresh", "regime-append", "regime-append"]
|
||||||
|
assert len(sent) == 3 and sum(b"handleCode=500" in x[1] for x in sent) == 1
|
||||||
|
assert not xxl._pending and xxl._running_id is None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- runner
|
# ---------------------------------------------------------------- runner
|
||||||
def main():
|
def main():
|
||||||
passed, failed = 0, 0
|
passed, failed = 0, 0
|
||||||
|
|
|
||||||
50
xxl.py
50
xxl.py
|
|
@ -18,7 +18,7 @@ tasks_periodic 的 notify_xxl),对平台表现完全一致:
|
||||||
但它派生的子进程会,和宿主 cron 的 docker exec 语义完全一致。
|
但它派生的子进程会,和宿主 cron 的 docker exec 语义完全一致。
|
||||||
|
|
||||||
安全默认与决策系统相同:.env 不配 XXL_TRIGGER_KEY,这一组端点整体禁用。
|
安全默认与决策系统相同:.env 不配 XXL_TRIGGER_KEY,这一组端点整体禁用。
|
||||||
同一时刻只允许一条链在跑:已有任务未结束时再触发回 409,由平台按失败重试策略处理。
|
同一时刻只执行一条链。完整修订之间有界串行排队,其他忙时请求仍回 409。
|
||||||
|
|
||||||
步骤白名单(steps 参数,逗号分隔,默认三步全跑,执行顺序固定):
|
步骤白名单(steps 参数,逗号分隔,默认三步全跑,执行顺序固定):
|
||||||
build 因子构建(run.py build all --mode daily)
|
build 因子构建(run.py build all --mode daily)
|
||||||
|
|
@ -75,6 +75,9 @@ STEP_TIMEOUT_SEC = 3600 # 单步上限一小时,防呆死(正常盘
|
||||||
_lock = threading.Lock()
|
_lock = threading.Lock()
|
||||||
_jobs: dict = {} # task_id -> 状态字典(常驻进程内存,重启即清空)
|
_jobs: dict = {} # task_id -> 状态字典(常驻进程内存,重启即清空)
|
||||||
_running_id = None
|
_running_id = None
|
||||||
|
_pending = [] # (task_id, steps, date, callback_url),只排完整修订
|
||||||
|
_REVISION_STEPS = frozenset({"plan-refresh", "regime-append"})
|
||||||
|
_MAX_PENDING = 2
|
||||||
_KEEP_JOBS = 20 # 只留最近 20 条状态,防内存慢涨
|
_KEEP_JOBS = 20 # 只留最近 20 条状态,防内存慢涨
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -161,10 +164,19 @@ def _run_job(task_id: str, steps: list, date, callback_url):
|
||||||
except Exception as e: # noqa: BLE001 —— 线程里绝不让异常无声消失
|
except Exception as e: # noqa: BLE001 —— 线程里绝不让异常无声消失
|
||||||
ok, note = False, f"意外异常: {type(e).__name__}: {e}"
|
ok, note = False, f"意外异常: {type(e).__name__}: {e}"
|
||||||
_log(job, note)
|
_log(job, note)
|
||||||
job.update(state="success" if ok else "failed", note=note,
|
next_job = None
|
||||||
ended_at=dt.datetime.now().isoformat(timespec="seconds"), step=None)
|
|
||||||
with _lock:
|
with _lock:
|
||||||
|
job.update(state="success" if ok else "failed", note=note,
|
||||||
|
ended_at=dt.datetime.now().isoformat(timespec="seconds"), step=None)
|
||||||
_running_id = None
|
_running_id = None
|
||||||
|
if _pending:
|
||||||
|
next_job = _pending.pop(0)
|
||||||
|
_running_id = next_job[0]
|
||||||
|
_jobs[_running_id].update(state="running",
|
||||||
|
started_at=dt.datetime.now().isoformat(timespec="seconds"))
|
||||||
|
# 回调的网络重试不能延迟下一个修订;此时前一个装配进程已结束。
|
||||||
|
if next_job:
|
||||||
|
threading.Thread(target=_run_job, args=next_job, daemon=True).start()
|
||||||
_notify(job, callback_url, ok, note)
|
_notify(job, callback_url, ok, note)
|
||||||
job["closed"] = True # 结案回调也处理完了(送达或放弃),状态接口可据此判断
|
job["closed"] = True # 结案回调也处理完了(送达或放弃),状态接口可据此判断
|
||||||
|
|
||||||
|
|
@ -197,29 +209,39 @@ def trigger_daily_build(
|
||||||
ordered = [s for s in STEP_ORDER if s in wanted]
|
ordered = [s for s in STEP_ORDER if s in wanted]
|
||||||
|
|
||||||
with _lock:
|
with _lock:
|
||||||
if _running_id and _jobs.get(_running_id, {}).get("state") == "running":
|
running = _jobs.get(_running_id, {})
|
||||||
|
busy = running.get("state") == "running"
|
||||||
|
if busy and not (set(ordered) <= _REVISION_STEPS
|
||||||
|
and set(running["steps"]) <= _REVISION_STEPS
|
||||||
|
and len(_pending) < _MAX_PENDING):
|
||||||
raise HTTPException(status_code=409,
|
raise HTTPException(status_code=409,
|
||||||
detail=f"已有任务在跑(task_id={_running_id}),本次拒绝")
|
detail=f"已有任务在跑或修订等待队列已满(task_id={_running_id}),本次拒绝")
|
||||||
task_id = str(uuid.uuid4())
|
task_id = str(uuid.uuid4())
|
||||||
_jobs[task_id] = {"task_id": task_id, "state": "running", "steps": ordered,
|
accepted_at = dt.datetime.now().isoformat(timespec="seconds")
|
||||||
|
_jobs[task_id] = {"task_id": task_id, "state": "queued" if busy else "running", "steps": ordered,
|
||||||
"step": None, "date": date, "note": "", "tail": [],
|
"step": None, "date": date, "note": "", "tail": [],
|
||||||
"started_at": dt.datetime.now().isoformat(timespec="seconds"),
|
"accepted_at": accepted_at, "started_at": None if busy else accepted_at,
|
||||||
"ended_at": None, "closed": False}
|
"ended_at": None, "closed": False}
|
||||||
while len(_jobs) > _KEEP_JOBS: # 只留最近 N 条
|
while len(_jobs) > _KEEP_JOBS:
|
||||||
oldest = next(iter(_jobs))
|
oldest = next((k for k, v in _jobs.items() if v.get("closed")), None)
|
||||||
if oldest == task_id:
|
if oldest is None:
|
||||||
break
|
break
|
||||||
_jobs.pop(oldest)
|
_jobs.pop(oldest)
|
||||||
_running_id = task_id
|
if busy:
|
||||||
|
_pending.append((task_id, ordered, date, callbackUrl))
|
||||||
|
_log(_jobs[task_id], "等待前一个完整修订结束")
|
||||||
|
else:
|
||||||
|
_running_id = task_id
|
||||||
|
|
||||||
threading.Thread(target=_run_job, args=(task_id, ordered, date, callbackUrl),
|
if not busy:
|
||||||
daemon=True).start()
|
threading.Thread(target=_run_job, args=(task_id, ordered, date, callbackUrl),
|
||||||
|
daemon=True).start()
|
||||||
return {"status": "accepted", "task_id": task_id, "steps": ordered, "date": date}
|
return {"status": "accepted", "task_id": task_id, "steps": ordered, "date": date}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/status/{task_id}")
|
@router.get("/status/{task_id}")
|
||||||
def job_status(task_id: str, key: str = Query(None), x_job_key: str = Header(None)):
|
def job_status(task_id: str, key: str = Query(None), x_job_key: str = Header(None)):
|
||||||
"""状态兜底:running / success / failed + 当前步骤 + 最近输出。"""
|
"""状态兜底:queued / running / success / failed + 当前步骤 + 最近输出。"""
|
||||||
_require_key(x_job_key, key)
|
_require_key(x_job_key, key)
|
||||||
job = _jobs.get(task_id)
|
job = _jobs.get(task_id)
|
||||||
if not job:
|
if not job:
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue