tradingSystem/app/services/plan_feed.py

# -*- coding: utf-8 -*-
"""
上游选股计划接入 (HTTP /plan) —— 候选池的事实源
================================================
上游「选股系统」2026-07-30 起提供计划接口, 取代原先从 `trading_buy_plan` 捞
`is_active=7` 的路子 (那张表是 trading_service 时代的产物, 目标架构下没有明确写入方)。

    GET {PMS_PLAN_API_BASE}{PMS_PLAN_API_PATH}?date=YYYY-MM-DD
    → {date, counts{main,observe,gate_covered}, theme_cap, heat_date,
       market_snapshot_days[], main:[...], observe:[...], changes, encoding}

主榜 main 每项: {rank, code("SH600418" 前缀式), name, score, heat, upside, tier,
                evidence{theme, n_sources, moved_ratio}}; 观察档 observe 同形但
                无 tier、upside 为 null。

五条口径在此钉死 (口径本身的疑问见 UPSTREAM_PLAN_API.md 的待确认项):

  1. **计划里没有价格、没有金额。** 上游只回答「买什么、排第几」, 买多少、什么价位是
     PMS 自己的活 (sizer/planner)。价格一律走 `market.get_price`, 取不到就不进池 ——
     绝不拿 score/upside 当价格用。
  2. **拿不到 ≠ 今天没票可买。** 超时、报错、JSON 变形、日期过期一律抛 `PlanFeedError`,
     由调用方记 ERROR 且让候选池为空, 绝不静默回退到旧表。沿用研判闸那条
     「UNAVAILABLE ≠ PASS」的纪律: 事实源缺失要显式失败, 不能被当成"没有候选"。
  3. **961 只主榜不是买入清单, 是排序池。** 按 score 降序取前 `PMS_PLAN_TOP_N`
     (默认 30) 进候选, 档位再由 `PMS_PLAN_TIERS` 白名单过滤。
  4. **日期新鲜度硬校验。** 计划日期比今天旧超过 `PMS_PLAN_STALE_TDAYS` 个交易日即
     过期 (抛错)。防的是节假日/上游停更时拿上周的榜当今天用 —— 这种错在盘中是静默的。
  5. **upside = 相对现价的预期空间比例** (2.1203 → +212%; 2026-07-30 与上游
     `format=md` 输出的「预期空间 +212%」对齐确认), 来源是券商目标价。噪音大 —— 榜首能
     到 +212% —— 所以**永不参与排序** (排序始终是 score), 只提供下限过滤
     `PMS_PLAN_MIN_UPSIDE`。

行业约束的数据源也从这里来: `evidence.theme` 随每次刷新 upsert 进 `pms_industry_map`,
行业源仍是 `custom_table` (`PMS_SECTOR_SOURCE=custom_table`)。落库而不是即时查, 是因为
即时查有个洞 —— 今天没上榜的持仓票查不到 theme, 行业硬拦截就对它悄悄失效了; 落库以后
覆盖面随时间累积, 且页面可见可手改。

模块级只依赖 stdlib + `app.core.command_spec` (纯逻辑), 其余 (requests / param_store /
pms_repo / tradedays) 一律函数内懒加载 —— 让解析与筛选这两段纯逻辑可以零依赖单测。
"""
from __future__ import annotations

import logging
import threading
import time

from app.core.command_spec import normalize_code

logger = logging.getLogger("pms.plan")

BUCKET_MAIN, BUCKET_OBSERVE = "main", "observe"

# 候选池来源 (PMS_CANDIDATE_SOURCE)
SRC_PLAN_API, SRC_BUY_PLAN, SRC_BOTH = "plan_api", "buy_plan", "both"

FAIL_CACHE_SEC = 60.0   # 失败也缓存一会儿, 免得每分钟的调度位把 10 秒超时叠成雪崩


class PlanFeedError(RuntimeError):
    """上游计划取不到 / 变形 / 过期。**必须**冒泡, 不得吞成空计划 (见头部口径 2)。"""


# ================================================================ 纯逻辑: 解析
def _int_or_none(v):
    try:
        if v is None or (isinstance(v, str) and not v.strip()):
            return None
        return int(float(v))
    except (TypeError, ValueError):
        return None


def _float_or_none(v):
    try:
        if v is None or (isinstance(v, str) and not v.strip()):
            return None
        return float(v)
    except (TypeError, ValueError):
        return None


def _text_or_none(v):
    if v is None:
        return None
    s = str(v).strip()
    return s or None


def _rows(raw, bucket: str) -> list:
    """一档榜单归一。坏行跳过而不是整体抛错 —— 单条变形不该让整张计划失效。"""
    out, seen = [], set()
    for i, it in enumerate(raw or []):
        if not isinstance(it, dict):
            continue
        code = normalize_code(str(it.get("code") or it.get("ts_code") or ""))
        if not code or code in seen:
            continue
        seen.add(code)
        ev = it.get("evidence") if isinstance(it.get("evidence"), dict) else {}
        rank = _int_or_none(it.get("rank"))
        out.append({
            "ts_code": code,
            "name": _text_or_none(it.get("name")),
            "rank": rank if rank is not None else i + 1,
            "score": _float_or_none(it.get("score")),
            "heat": _float_or_none(it.get("heat")),
            "upside": _float_or_none(it.get("upside")),      # 相对现价的比例, 2.12=+212%
            "tier": _text_or_none(it.get("tier")),
            "theme": _text_or_none(ev.get("theme")),
            "n_sources": _int_or_none(ev.get("n_sources")),
            "moved_ratio": _float_or_none(ev.get("moved_ratio")),
            "bucket": bucket,
        })
    return out


def parse_plan(payload) -> dict:
    """应答 → 内部结构。缺 date 或两档全空都算变形 (抛 PlanFeedError)。"""
    if not isinstance(payload, dict):
        raise PlanFeedError(f"应答不是 JSON 对象: {type(payload).__name__}")
    date = _text_or_none(payload.get("date"))
    if not date:
        raise PlanFeedError("应答缺 date —— 判不了新鲜度, 按取不到处理")
    main = _rows(payload.get("main"), BUCKET_MAIN)
    observe = _rows(payload.get("observe"), BUCKET_OBSERVE)
    if not main and not observe:
        raise PlanFeedError(f"计划 {date} 主榜与观察档都是空的")
    counts = payload.get("counts") if isinstance(payload.get("counts"), dict) else {}
    themes = {}
    for r in main + observe:            # 主榜在前, 同码以主榜的 theme 为准
        if r["theme"] and r["ts_code"] not in themes:
            themes[r["ts_code"]] = r["theme"]
    return {
        "date": date,
        "heat_date": _text_or_none(payload.get("heat_date")),
        "market_snapshot_days": [str(x) for x in (payload.get("market_snapshot_days") or [])],
        "theme_cap": _int_or_none(payload.get("theme_cap")),
        "encoding": _text_or_none(payload.get("encoding")),
        "counts": {"main": _int_or_none(counts.get("main")),
                   "observe": _int_or_none(counts.get("observe")),
                   "gate_covered": _int_or_none(counts.get("gate_covered"))},
        "returned": {"main": len(main), "observe": len(observe)},
        "main": main, "observe": observe, "themes": themes,
    }


# ================================================================ 纯逻辑: 新鲜度
def plan_age_tdays(plan_date, today=None) -> int:
    """计划日期距今的交易日龄。当天=0, 上一个交易日=1; 日期在未来 (为下一交易日出的计划)=0。"""
    from app.core import tradedays as td
    left = td.trade_days_left(today, plan_date)     # 含首尾; plan_date 晚于 today 时为 0
    return max(0, left - 1)


def assert_fresh(plan: dict, *, max_stale_tdays: int = 1, today=None) -> int:
    age = plan_age_tdays(plan["date"], today)
    if age > int(max_stale_tdays or 0):
        raise PlanFeedError(
            f"上游计划已过期: 日期 {plan['date']} 距今 {age} 个交易日 "
            f"(上限 {max_stale_tdays})。上游可能停更 —— 拿旧榜当今天用比没有候选更危险")
    return age


# ================================================================ 纯逻辑: 筛选
def select_candidates(plan: dict, *, held=(), black=(), top_n: int = 30, tiers=None,
                      include_observe: bool = False, min_score=None,
                      min_sources: int = 0, min_upside=None) -> dict:
    """排序池 → 候选清单。

    排序: score 降序, 同分按 rank 升序 (上游 rank 已是它自己的最终次序, 拿来当稳定次序)。
    tier 白名单**只对带 tier 的行生效** —— 观察档没有 tier, 它的闸门是 include_observe。
    min_upside 相反, **对所有行生效**: upside 缺失按 0 算一起挡掉 (观察档 upside 恒为
    null, 所以设了下限等于把观察档全挡了)。方向选保守那边 —— 宁可少票。
    """
    held = {normalize_code(c) for c in (held or []) if c}
    black = {normalize_code(c) for c in (black or []) if c}
    tiers = {str(t).strip() for t in (tiers or []) if str(t).strip()}
    min_score = _float_or_none(min_score)
    min_upside = _float_or_none(min_upside)
    min_sources = int(min_sources or 0)

    pool = list(plan.get("main") or [])
    if include_observe:
        pool += list(plan.get("observe") or [])

    dropped = {"held": 0, "black": 0, "tier": 0, "score": 0, "sources": 0, "upside": 0,
               "dup": 0, "capped": 0}
    passed, seen = [], set()
    for r in sorted(pool, key=lambda x: (-(x.get("score") or 0.0), x.get("rank") or 10 ** 9)):
        c = r["ts_code"]
        if c in seen:
            dropped["dup"] += 1
            continue
        seen.add(c)
        if c in held:
            dropped["held"] += 1
            continue
        if c in black:
            dropped["black"] += 1
            continue
        if tiers and r.get("tier") is not None and r["tier"] not in tiers:
            dropped["tier"] += 1
            continue
        if min_score is not None and (r.get("score") or 0.0) < min_score:
            dropped["score"] += 1
            continue
        if min_sources and (r.get("n_sources") or 0) < min_sources:
            dropped["sources"] += 1
            continue
        if min_upside is not None and (r.get("upside") or 0.0) < min_upside:
            dropped["upside"] += 1
            continue
        passed.append(r)

    n = max(0, int(top_n or 0)) or len(passed)
    dropped["capped"] = max(0, len(passed) - n)
    items = [{"ts_code": r["ts_code"], "name": r["name"], "score": r.get("score") or 0.0,
              "sector": r.get("theme"), "theme": r.get("theme"), "tier": r.get("tier"),
              "heat": r.get("heat"), "upside": r.get("upside"), "rank": r.get("rank"),
              "bucket": r["bucket"], "src": "plan_api"}
             for r in passed[:n]]
    return {"date": plan.get("date"), "considered": len(pool), "eligible": len(passed),
            "items": items, "dropped": dropped}


# ================================================================ 取数 (带缓存)
_cache = {"at": 0.0, "key": None, "plan": None, "error": None}
_lock = threading.Lock()


def _params() -> dict:
    from app.services import param_store as ps
    return {
        "base": (ps.get("PMS_PLAN_API_BASE", "") or "").strip().rstrip("/"),
        "path": (ps.get("PMS_PLAN_API_PATH", "/plan") or "/plan").strip(),
        "timeout": ps.get_int("PMS_PLAN_TIMEOUT", 10),
        "cache_sec": ps.get_int("PMS_PLAN_CACHE_SEC", 300),
        "top_n": ps.get_int("PMS_PLAN_TOP_N", 30),
        "tiers": ps.get_list("PMS_PLAN_TIERS", []),
        "include_observe": ps.get_bool("PMS_PLAN_INCLUDE_OBSERVE", False),
        "min_score": ps.get_float("PMS_PLAN_MIN_SCORE", 0.0),
        "min_sources": ps.get_int("PMS_PLAN_MIN_SOURCES", 0),
        "min_upside": ps.get_float("PMS_PLAN_MIN_UPSIDE", 0.0),
        "stale_tdays": ps.get_int("PMS_PLAN_STALE_TDAYS", 1),
        "theme_sync": ps.get_bool("PMS_PLAN_THEME_SYNC", True),
        "source": (ps.get("PMS_CANDIDATE_SOURCE", SRC_PLAN_API) or SRC_PLAN_API).strip(),
    }


def enabled() -> bool:
    return bool(_params()["base"])


def fetch(*, date=None, base=None, path=None, timeout=None) -> dict:
    """拉一次并解析 (不走缓存、不校新鲜度)。任何失败抛 PlanFeedError。

    三个参数全给齐时**不读参数中心** —— 纯取数函数不该依赖 ParamStore, 否则连
    「base 为空立即失败」这条守卫都没法脱库单测。
    """
    if base is None or path is None or timeout is None:
        p = _params()
        base = p["base"] if base is None else base
        path = p["path"] if path is None else path
        timeout = p["timeout"] if timeout is None else timeout
    base = (base or "").strip().rstrip("/")
    if not base:
        raise PlanFeedError("上游计划接口未配置 (PMS_PLAN_API_BASE 为空)")
    path = (path or "/plan").strip() or "/plan"
    if not path.startswith("/"):
        path = "/" + path
    url = base + path
    to = int(timeout or 10)
    try:
        import requests
        r = requests.get(url, params=({"date": date} if date else None), timeout=to)
        r.raise_for_status()
        payload = r.json()
    except PlanFeedError:
        raise
    except Exception as e:
        raise PlanFeedError(f"拉取上游计划失败 {url}: {type(e).__name__}: {e}") from e
    plan = parse_plan(payload)
    plan["url"] = url
    plan["fetched_at"] = time.time()
    plan["requested_date"] = date
    return plan


def get_plan(*, force: bool = False, date=None) -> dict:
    """带缓存的当前计划。失败同样缓存 FAIL_CACHE_SEC, 但每次调用都照样抛。"""
    p = _params()
    key = (p["base"], p["path"], date or "")
    now = time.time()
    with _lock:
        fresh_hit = (not force and _cache["key"] == key and _cache["plan"] is not None
                     and now - _cache["at"] < max(1, p["cache_sec"]))
        if fresh_hit:
            return _cache["plan"]
        if (not force and _cache["key"] == key and _cache["error"]
                and now - _cache["at"] < FAIL_CACHE_SEC):
            raise PlanFeedError(_cache["error"])
    try:
        plan = fetch(date=date)
        assert_fresh(plan, max_stale_tdays=p["stale_tdays"])
    except PlanFeedError as e:
        with _lock:
            _cache.update({"at": time.time(), "key": key, "plan": None, "error": str(e)})
        raise
    plan["age_tdays"] = plan_age_tdays(plan["date"])
    if p["theme_sync"]:
        plan["theme_sync"] = _sync_themes_quiet(plan)
    with _lock:
        _cache.update({"at": time.time(), "key": key, "plan": plan, "error": None})
    logger.info("[上游计划] %s 主榜 %d / 观察 %d (日龄 %d 交易日) ← %s",
                plan["date"], plan["returned"]["main"], plan["returned"]["observe"],
                plan["age_tdays"], plan["url"])
    return plan


def invalidate():
    with _lock:
        _cache.update({"at": 0.0, "key": None, "plan": None, "error": None})


# ================================================================ theme → 行业映射
def sync_themes(plan: dict) -> dict:
    """把 code→theme upsert 进 pms_industry_map (行业源 custom_table 的灌数口径)。"""
    from app.repo import pms_repo
    from app.services import industry
    rows = [{"ts_code": c, "industry": t} for c, t in (plan.get("themes") or {}).items()]
    if not rows:
        return {"rows": 0, "affected": 0}
    n = pms_repo.upsert_industry(rows)
    industry.invalidate()
    return {"rows": len(rows), "affected": n}


def _sync_themes_quiet(plan: dict) -> dict:
    """theme 落库失败不许阻断候选池 —— 行业约束停用是可接受的降级, 没候选不是。"""
    try:
        return sync_themes(plan)
    except Exception as e:
        logger.warning("[上游计划] theme 落 pms_industry_map 失败 (行业约束按未配置降级): %s", e)
        return {"rows": 0, "affected": 0, "error": f"{type(e).__name__}: {e}"}


# ================================================================ 对外: 候选与状态
def candidates(*, held=(), black=()) -> dict:
    """参数驱动的候选清单 (不含价格 —— 价格由调用方用 market.get_price 现取)。"""
    p = _params()
    plan = get_plan()
    out = select_candidates(plan, held=held, black=black, top_n=p["top_n"], tiers=p["tiers"],
                            include_observe=p["include_observe"],
                            min_score=(p["min_score"] or None),
                            min_sources=p["min_sources"],
                            min_upside=(p["min_upside"] or None))
    out["age_tdays"] = plan.get("age_tdays")
    out["theme_cap"] = plan.get("theme_cap")
    return out


def status() -> dict:
    """页面/运维用: 绝不抛错, 失败也要能显示出来。"""
    p = _params()
    st = {"source": p["source"], "base": p["base"], "path": p["path"],
          "top_n": p["top_n"], "tiers": p["tiers"], "min_upside": p["min_upside"],
          "include_observe": p["include_observe"], "stale_tdays": p["stale_tdays"],
          "theme_sync": p["theme_sync"], "enabled": bool(p["base"])}
    if not p["base"]:
        st.update({"ok": False, "hint": "上游计划接口未配置 (PMS_PLAN_API_BASE 为空) —— "
                                        "候选池将为空, 升仓/建仓类命令无票可选"})
        return st
    try:
        plan = get_plan()
    except PlanFeedError as e:
        st.update({"ok": False, "hint": str(e)})
        return st
    except Exception as e:                      # 兜底: status 不许抛
        st.update({"ok": False, "hint": f"{type(e).__name__}: {e}"})
        return st
    st.update({"ok": True, "date": plan["date"], "age_tdays": plan.get("age_tdays"),
               "heat_date": plan.get("heat_date"),
               "market_snapshot_days": plan.get("market_snapshot_days"),
               "counts": plan["counts"], "returned": plan["returned"],
               "theme_cap": plan.get("theme_cap"), "encoding": plan.get("encoding"),
               "theme_sync": plan.get("theme_sync"),
               "fetched_at": plan.get("fetched_at"), "url": plan.get("url"),
               "hint": f"计划 {plan['date']} 已就绪 (日龄 {plan.get('age_tdays')} 交易日)"})
    return st