diff --git a/docs/复盘决定台账.md b/docs/复盘决定台账.md index 63729c9..f7b709e 100644 --- a/docs/复盘决定台账.md +++ b/docs/复盘决定台账.md @@ -683,3 +683,12 @@ - **067 部署补记。** e44f770 本地及候选镜像各 36 组禁网回归通过,10-16 07:33:40 桥重建生效。今晨版本原字节保持,晚生成 10-14 样本已排除,旧格式原件未恢复。07:35 XXL 137 首次自动检查成功,07:35:36 155 提议重核已开启,其他参数保持。08:55 接续实际消费与区制修订,无额外五日观察。证据与不可恢复边界见同日交接 10-16 追加节。 + + +## 068 · 2026-10-16 · 区制与五分钟修订有界串行受理 + +- **日期。** 2026-10-16,北京时间。 +- **改动。** 两类完整修订同刻触发时进入最多两条的等待队列,前一完成或失败均继续下一条,各自结案。普通构建继续忙时拒绝。 +- **依据。** 08:45 XXL 122 被同刻 137 占用拒绝,日志 92246 返回 409,区制未追加。两个方向、故障释放与队列上限加入回归。 +- **预期。** 区制与评析按顺序发布完整版本,原晨版和归档不变。原调度时刻、排序、参数与提议规则不变。 +- **复核日期。** 10-16 本地与镜像禁网测试后部署,只补执行原区制任务;通过正常盘中消费继续判收。 diff --git a/test_regime.py b/test_regime.py index 0760e27..2259da9 100644 --- a/test_regime.py +++ b/test_regime.py @@ -1,7 +1,7 @@ """regime.py 离线单测(无需网络、无需库):未配置 -> UNKNOWN;两种快照形状都能归一; 弱势日按旋钮判;追加进临时快照并能读回。 -跑法:python3 test_regime.py 或 pytest test_regime.py +跑法:python3 test_regime.py;依赖使用项目环境,网络与来源均为替身。 """ import io import json @@ -78,10 +78,36 @@ def main(): and not os.path.exists(regime.snapshot_path("2026-09-01"))) with open(regime.snapshot_path("2026-09-01"), "w", encoding="utf-8") as f: json.dump({"date": "2026-09-01", "main": []}, f) - regime.append_to_snapshot("2026-09-01") - back = regime.read_from_snapshot("2026-09-01") - t("追加后能从快照读回 regime 段", back and back["status"] == "OK" and back["weak_count"] == 3) - t("快照其余内容不丢", json.load(open(regime.snapshot_path("2026-09-01"), encoding="utf-8"))["main"] == []) + import datetime as dt + from pathlib import Path + from unittest.mock import patch + import plan, plan_refresh, plan_versions as pv + now = dt.datetime(2026, 9, 2, 0, 45, tzinfo=dt.timezone.utc) + actual_refresh = plan_refresh.refresh + def at_fixed_time(*args, **kwargs): + return actual_refresh(*args, now=now, **kwargs) + path = Path(regime.snapshot_path("2026-09-01")) + legacy = path.read_bytes() + with patch('plan_refresh.refresh', side_effect=at_fixed_time): + regime.append_to_snapshot("2026-09-01") + t("旧格式不补造完整晨版,原字节保持", path.read_bytes() == legacy) + path.unlink() # 仅删除测试临时夹具,另起完整晨版场景。 + morning = {'date': '2026-09-01', 'plan_version': 'test', 'main': [], 'observe': []} + with pv.writer(d): + old = pv.publish(d, morning, {'records': {}, 'market': None, 'regime': None}, {}, + cutoff_at='2026-09-01T23:10:00+00:00', reason='晨版生成') + archive = Path(d)/'versions'/'2026-09-01'/(old['decision']['id']+'.json') + before = archive.read_bytes() + def assembled(*args, **kwargs): + return dict(morning, _full={'main': [], 'observe': []}) + with patch.object(plan, 'collect', side_effect=assembled), patch.object(plan, 'write_published_md'): + regime.append_to_snapshot('2026-09-01') + back = regime.read_from_snapshot('2026-09-01') + t("追加后完整版本可读区制", back and back['status']=='OK' and back['weak_count']==3) + t("晨版固定且旧归档不变", archive.read_bytes()==before and pv.load(d,'2026-09-01',morning=True)==old) + t("发布独立完整修订", len(pv.history(d,'2026-09-01'))==2) + regime.append_to_snapshot('2026-09-01') + t("同区制重复追加不发布空版本", len(pv.history(d,'2026-09-01'))==2) finally: urllib.request.urlopen = orig print("ALL OK — 环境标签:未配置 / 两种形状 / 弱势日旋钮 / 关闭 / 不可达 / 快照追加 全部通过") diff --git a/test_xxl_trigger.py b/test_xxl_trigger.py index 8a5fc20..c299704 100644 --- a/test_xxl_trigger.py +++ b/test_xxl_trigger.py @@ -29,6 +29,7 @@ def _reset(key="test-key"): config.XXL_CALLBACK_MAX_RETRIES = 0 xxl._jobs.clear() xxl._running_id = None + xxl._pending.clear() ran, sent = [], [] xxl._run_step = lambda cmd: (ran.append(list(cmd)), (0, "ok"))[1] xxl._post_callback = lambda url, data: (sent.append((url, data)), "delivered http=200")[1] @@ -185,6 +186,76 @@ def _(): assert any("放弃" in ln for ln in job["tail"]), job["tail"] + +@case("评析和区制同时触发:两方向串行,各自结案一次") +def _(): + import threading + for first, second in [("plan-refresh", "regime-append"), ("regime-append", "plan-refresh")]: + ran, sent = _reset() + release = threading.Event() + entered = threading.Event() + def slow(cmd): + ran.append(cmd[1]) + if len(ran) == 1: + entered.set() + assert release.wait(3) + return 0, "ok" + xxl._run_step = slow + a = _trigger(steps=first, callbackUrl="http://xxl/a") + assert entered.wait(2) + try: + b = _trigger(steps=second, callbackUrl="http://xxl/b") + assert xxl._jobs[b["task_id"]]["state"] == "queued" + assert xxl._jobs[b["task_id"]]["started_at"] is None + assert ran == [first] + finally: + release.set() + assert _wait_done(a["task_id"])["state"] == "success" + assert _wait_done(b["task_id"])["state"] == "success" + assert ran == [first, second] + assert sorted(x[0] for x in sent) == ["http://xxl/a", "http://xxl/b"] + assert not xxl._pending and xxl._running_id is None + + +@case("修订失败后放行等待项,队列有界且拒绝混入普通构建") +def _(): + import threading + ran, sent = _reset() + release = threading.Event() + entered = threading.Event() + def slow(cmd): + ran.append(cmd[1]) + if len(ran) == 1: + entered.set() + assert release.wait(3) + return 1, "source unavailable" + return 0, "ok" + xxl._run_step = slow + a = _trigger(steps="plan-refresh", callbackUrl="http://xxl/a") + assert entered.wait(2) + try: + for step in ["build", "plan-refresh,build"]: + try: + _trigger(steps=step) + raise AssertionError("普通构建不得进入修订队列") + except HTTPException as e: + assert e.status_code == 409 + bs = [_trigger(steps="regime-append", callbackUrl=f"http://xxl/b{i}") for i in range(2)] + try: + _trigger(steps="plan-refresh") + raise AssertionError("等待队列不得无限增长") + except HTTPException as e: + assert e.status_code == 409 + finally: + release.set() + assert _wait_done(a["task_id"])["state"] == "failed" + for b in bs: + assert _wait_done(b["task_id"])["state"] == "success" + assert ran == ["plan-refresh", "regime-append", "regime-append"] + assert len(sent) == 3 and sum(b"handleCode=500" in x[1] for x in sent) == 1 + assert not xxl._pending and xxl._running_id is None + + # ---------------------------------------------------------------- runner def main(): passed, failed = 0, 0 diff --git a/xxl.py b/xxl.py index 07ee38b..46dfed9 100644 --- a/xxl.py +++ b/xxl.py @@ -18,7 +18,7 @@ tasks_periodic 的 notify_xxl),对平台表现完全一致: 但它派生的子进程会,和宿主 cron 的 docker exec 语义完全一致。 安全默认与决策系统相同:.env 不配 XXL_TRIGGER_KEY,这一组端点整体禁用。 -同一时刻只允许一条链在跑:已有任务未结束时再触发回 409,由平台按失败重试策略处理。 +同一时刻只执行一条链。完整修订之间有界串行排队,其他忙时请求仍回 409。 步骤白名单(steps 参数,逗号分隔,默认三步全跑,执行顺序固定): build 因子构建(run.py build all --mode daily) @@ -75,6 +75,9 @@ STEP_TIMEOUT_SEC = 3600 # 单步上限一小时,防呆死(正常盘 _lock = threading.Lock() _jobs: dict = {} # task_id -> 状态字典(常驻进程内存,重启即清空) _running_id = None +_pending = [] # (task_id, steps, date, callback_url),只排完整修订 +_REVISION_STEPS = frozenset({"plan-refresh", "regime-append"}) +_MAX_PENDING = 2 _KEEP_JOBS = 20 # 只留最近 20 条状态,防内存慢涨 @@ -161,10 +164,19 @@ def _run_job(task_id: str, steps: list, date, callback_url): except Exception as e: # noqa: BLE001 —— 线程里绝不让异常无声消失 ok, note = False, f"意外异常: {type(e).__name__}: {e}" _log(job, note) - job.update(state="success" if ok else "failed", note=note, - ended_at=dt.datetime.now().isoformat(timespec="seconds"), step=None) + next_job = None with _lock: + job.update(state="success" if ok else "failed", note=note, + ended_at=dt.datetime.now().isoformat(timespec="seconds"), step=None) _running_id = None + if _pending: + next_job = _pending.pop(0) + _running_id = next_job[0] + _jobs[_running_id].update(state="running", + started_at=dt.datetime.now().isoformat(timespec="seconds")) + # 回调的网络重试不能延迟下一个修订;此时前一个装配进程已结束。 + if next_job: + threading.Thread(target=_run_job, args=next_job, daemon=True).start() _notify(job, callback_url, ok, note) job["closed"] = True # 结案回调也处理完了(送达或放弃),状态接口可据此判断 @@ -197,29 +209,39 @@ def trigger_daily_build( ordered = [s for s in STEP_ORDER if s in wanted] with _lock: - if _running_id and _jobs.get(_running_id, {}).get("state") == "running": + running = _jobs.get(_running_id, {}) + busy = running.get("state") == "running" + if busy and not (set(ordered) <= _REVISION_STEPS + and set(running["steps"]) <= _REVISION_STEPS + and len(_pending) < _MAX_PENDING): raise HTTPException(status_code=409, - detail=f"已有任务在跑(task_id={_running_id}),本次拒绝") + detail=f"已有任务在跑或修订等待队列已满(task_id={_running_id}),本次拒绝") task_id = str(uuid.uuid4()) - _jobs[task_id] = {"task_id": task_id, "state": "running", "steps": ordered, + accepted_at = dt.datetime.now().isoformat(timespec="seconds") + _jobs[task_id] = {"task_id": task_id, "state": "queued" if busy else "running", "steps": ordered, "step": None, "date": date, "note": "", "tail": [], - "started_at": dt.datetime.now().isoformat(timespec="seconds"), + "accepted_at": accepted_at, "started_at": None if busy else accepted_at, "ended_at": None, "closed": False} - while len(_jobs) > _KEEP_JOBS: # 只留最近 N 条 - oldest = next(iter(_jobs)) - if oldest == task_id: + while len(_jobs) > _KEEP_JOBS: + oldest = next((k for k, v in _jobs.items() if v.get("closed")), None) + if oldest is None: break _jobs.pop(oldest) - _running_id = task_id + if busy: + _pending.append((task_id, ordered, date, callbackUrl)) + _log(_jobs[task_id], "等待前一个完整修订结束") + else: + _running_id = task_id - threading.Thread(target=_run_job, args=(task_id, ordered, date, callbackUrl), - daemon=True).start() + if not busy: + threading.Thread(target=_run_job, args=(task_id, ordered, date, callbackUrl), + daemon=True).start() return {"status": "accepted", "task_id": task_id, "steps": ordered, "date": date} @router.get("/status/{task_id}") def job_status(task_id: str, key: str = Query(None), x_job_key: str = Header(None)): - """状态兜底:running / success / failed + 当前步骤 + 最近输出。""" + """状态兜底:queued / running / success / failed + 当前步骤 + 最近输出。""" _require_key(x_job_key, key) job = _jobs.get(task_id) if not job: