344 lines
12 KiB
Python
344 lines
12 KiB
Python
|
# coding:utf-8
|
|||
|
|
"""
|
|||
|
|
个股资金流向（同花顺）采集工具
|
|||
|
|
功能：从 Tushare 获取 moneyflow_ths 数据并落库
|
|||
|
|
API 文档: https://tushare.pro/document/2?doc_id=348
|
|||
|
|
说明：接口每日盘后更新，需至少 5000 积分
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import os
|
|||
|
|
import sys
|
|||
|
|
from datetime import datetime
|
|||
|
|
from typing import Optional
|
|||
|
|
|
|||
|
|
import pandas as pd
|
|||
|
|
import tushare as ts
|
|||
|
|
from sqlalchemy import create_engine, text
|
|||
|
|
|
|||
|
|
# 添加项目根目录到路径，确保能够读取配置
|
|||
|
|
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|||
|
|
sys.path.append(PROJECT_ROOT)
|
|||
|
|
|
|||
|
|
from src.scripts.config import TUSHARE_TOKEN
|
|||
|
|
|
|||
|
|
|
|||
|
|
class MoneyflowTHSCollector:
|
|||
|
|
"""个股资金流向（THS）采集器"""
|
|||
|
|
|
|||
|
|
def __init__(self, db_url: str, tushare_token: str, table_name: str = "gp_moneyflow_ths"):
|
|||
|
|
"""
|
|||
|
|
Args:
|
|||
|
|
db_url: 数据库连接 URL
|
|||
|
|
tushare_token: Tushare Token
|
|||
|
|
table_name: 目标表名，默认 gp_moneyflow_ths
|
|||
|
|
"""
|
|||
|
|
self.engine = create_engine(
|
|||
|
|
db_url,
|
|||
|
|
pool_size=5,
|
|||
|
|
max_overflow=10,
|
|||
|
|
pool_recycle=3600,
|
|||
|
|
)
|
|||
|
|
self.table_name = table_name
|
|||
|
|
|
|||
|
|
ts.set_token(tushare_token)
|
|||
|
|
self.pro = ts.pro_api()
|
|||
|
|
|
|||
|
|
print("=" * 60)
|
|||
|
|
print("个股资金流向（THS）采集工具")
|
|||
|
|
print(f"目标数据表: {self.table_name}")
|
|||
|
|
print("=" * 60)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def convert_tushare_code_to_db(ts_code: str) -> str:
|
|||
|
|
"""
|
|||
|
|
将 Tushare 代码（600000.SH）转换为数据库代码（SH600000）
|
|||
|
|
"""
|
|||
|
|
if not ts_code or "." not in ts_code:
|
|||
|
|
return ts_code
|
|||
|
|
base, market = ts_code.split(".")
|
|||
|
|
return f"{market}{base}"
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def convert_db_code_to_tushare(symbol: str) -> str:
|
|||
|
|
"""
|
|||
|
|
将数据库代码（SH600000）转换为 Tushare 代码（600000.SH）
|
|||
|
|
"""
|
|||
|
|
if not symbol:
|
|||
|
|
return symbol
|
|||
|
|
prefix = symbol[:2]
|
|||
|
|
code = symbol[2:]
|
|||
|
|
if prefix in {"SH", "SZ", "BJ"}:
|
|||
|
|
return f"{code}.{prefix}"
|
|||
|
|
return symbol
|
|||
|
|
|
|||
|
|
def fetch_data(
|
|||
|
|
self,
|
|||
|
|
ts_code: Optional[str] = None,
|
|||
|
|
trade_date: Optional[str] = None,
|
|||
|
|
start_date: Optional[str] = None,
|
|||
|
|
end_date: Optional[str] = None,
|
|||
|
|
) -> pd.DataFrame:
|
|||
|
|
"""
|
|||
|
|
调用 Tushare moneyflow_ths 接口获取数据
|
|||
|
|
Args 参数同接口文档
|
|||
|
|
"""
|
|||
|
|
try:
|
|||
|
|
df = self.pro.moneyflow_ths(
|
|||
|
|
ts_code=ts_code,
|
|||
|
|
trade_date=trade_date,
|
|||
|
|
start_date=start_date,
|
|||
|
|
end_date=end_date,
|
|||
|
|
)
|
|||
|
|
return df
|
|||
|
|
except Exception as exc:
|
|||
|
|
print(f"Tushare moneyflow_ths 调用失败: {exc}")
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
|
|||
|
|
def transform_data(self, df: pd.DataFrame) -> pd.DataFrame:
|
|||
|
|
"""
|
|||
|
|
将 Tushare 返回的数据转换为数据库入库格式
|
|||
|
|
"""
|
|||
|
|
if df.empty:
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
|
|||
|
|
result = pd.DataFrame()
|
|||
|
|
result["symbol"] = df["ts_code"].apply(self.convert_tushare_code_to_db)
|
|||
|
|
result["ts_code"] = df["ts_code"]
|
|||
|
|
result["trade_date"] = pd.to_datetime(df["trade_date"], format="%Y%m%d")
|
|||
|
|
result["name"] = df.get("name")
|
|||
|
|
result["pct_change"] = df.get("pct_change")
|
|||
|
|
result["latest"] = df.get("latest")
|
|||
|
|
result["net_amount"] = df.get("net_amount")
|
|||
|
|
result["net_d5_amount"] = df.get("net_d5_amount")
|
|||
|
|
result["buy_lg_amount"] = df.get("buy_lg_amount")
|
|||
|
|
result["buy_lg_amount_rate"] = df.get("buy_lg_amount_rate")
|
|||
|
|
result["buy_md_amount"] = df.get("buy_md_amount")
|
|||
|
|
result["buy_md_amount_rate"] = df.get("buy_md_amount_rate")
|
|||
|
|
result["buy_sm_amount"] = df.get("buy_sm_amount")
|
|||
|
|
result["buy_sm_amount_rate"] = df.get("buy_sm_amount_rate")
|
|||
|
|
result["created_at"] = datetime.now()
|
|||
|
|
result["updated_at"] = datetime.now()
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
def save_dataframe(self, df: pd.DataFrame) -> None:
|
|||
|
|
"""
|
|||
|
|
将数据写入数据库
|
|||
|
|
"""
|
|||
|
|
if df.empty:
|
|||
|
|
print("无数据需要保存")
|
|||
|
|
return
|
|||
|
|
df.to_sql(self.table_name, self.engine, if_exists="append", index=False)
|
|||
|
|
print(f"成功写入 {len(df)} 条记录")
|
|||
|
|
|
|||
|
|
def delete_by_date(self, date_str: str) -> None:
|
|||
|
|
"""
|
|||
|
|
删除指定交易日的旧数据
|
|||
|
|
"""
|
|||
|
|
with self.engine.begin() as conn:
|
|||
|
|
delete_sql = text(f"DELETE FROM {self.table_name} WHERE trade_date = :trade_date")
|
|||
|
|
affected = conn.execute(delete_sql, {"trade_date": date_str})
|
|||
|
|
print(f"已删除 {date_str} 旧数据 {affected.rowcount} 条")
|
|||
|
|
|
|||
|
|
def run_daily_collection(self, date: Optional[str] = None) -> None:
|
|||
|
|
"""
|
|||
|
|
获取指定交易日（默认当天）的全市场资金流向数据
|
|||
|
|
"""
|
|||
|
|
target_date = date or datetime.now().strftime("%Y-%m-%d")
|
|||
|
|
trade_date = datetime.strptime(target_date, "%Y-%m-%d").strftime("%Y%m%d")
|
|||
|
|
|
|||
|
|
print(f"开始采集 {target_date} ({trade_date}) 个股资金流向数据")
|
|||
|
|
self.delete_by_date(target_date)
|
|||
|
|
|
|||
|
|
df = self.fetch_data(trade_date=trade_date)
|
|||
|
|
if df.empty:
|
|||
|
|
print("接口未返回数据（可能为非交易日或权限不足）")
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
result_df = self.transform_data(df)
|
|||
|
|
if result_df.empty:
|
|||
|
|
print("数据转换失败")
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
self.save_dataframe(result_df)
|
|||
|
|
print("每日采集完成")
|
|||
|
|
|
|||
|
|
def run_full_collection(self) -> None:
|
|||
|
|
"""
|
|||
|
|
执行全量覆盖采集：
|
|||
|
|
- 清空目标表
|
|||
|
|
- 遍历数据库中所有股票代码，逐只拉取全历史数据
|
|||
|
|
"""
|
|||
|
|
print("=" * 60)
|
|||
|
|
print("开始执行全量覆盖采集（moneyflow_ths）")
|
|||
|
|
print("=" * 60)
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 清空表
|
|||
|
|
with self.engine.begin() as conn:
|
|||
|
|
conn.execute(text(f"TRUNCATE TABLE {self.table_name}"))
|
|||
|
|
print(f"{self.table_name} 已清空")
|
|||
|
|
|
|||
|
|
# 获取全部股票代码
|
|||
|
|
codes_df = pd.read_sql("SELECT gp_code FROM gp_code_all_copy", self.engine)
|
|||
|
|
codes = codes_df["gp_code"].tolist()
|
|||
|
|
print(f"共获取到 {len(codes)} 只股票")
|
|||
|
|
|
|||
|
|
total_records = 0
|
|||
|
|
success_count = 0
|
|||
|
|
failed_count = 0
|
|||
|
|
|
|||
|
|
for symbol in codes:
|
|||
|
|
ts_code = self.convert_db_code_to_tushare(symbol)
|
|||
|
|
try:
|
|||
|
|
df = self.fetch_data(ts_code=ts_code)
|
|||
|
|
if df.empty:
|
|||
|
|
failed_count += 1
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
result_df = self.transform_data(df)
|
|||
|
|
if result_df.empty:
|
|||
|
|
failed_count += 1
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
self.save_dataframe(result_df)
|
|||
|
|
total_records += len(result_df)
|
|||
|
|
success_count += 1
|
|||
|
|
except Exception as exc:
|
|||
|
|
print(f"\n采集 {symbol} 失败: {exc}")
|
|||
|
|
failed_count += 1
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
print("=" * 60)
|
|||
|
|
print("全量覆盖采集完成")
|
|||
|
|
print(f"总股票数: {len(codes)}")
|
|||
|
|
print(f"成功采集: {success_count}")
|
|||
|
|
print(f"失败: {failed_count}")
|
|||
|
|
print(f"累计记录: {total_records}")
|
|||
|
|
print("=" * 60)
|
|||
|
|
except Exception as exc:
|
|||
|
|
print(f"全量采集失败: {exc}")
|
|||
|
|
finally:
|
|||
|
|
self.engine.dispose()
|
|||
|
|
|
|||
|
|
def run_range_collection(self, start_date: str, end_date: str) -> None:
|
|||
|
|
"""
|
|||
|
|
获取时间区间内的全市场资金流向数据（按天分批）
|
|||
|
|
"""
|
|||
|
|
start = datetime.strptime(start_date, "%Y-%m-%d")
|
|||
|
|
end = datetime.strptime(end_date, "%Y-%m-%d")
|
|||
|
|
if start > end:
|
|||
|
|
raise ValueError("start_date 不能大于 end_date")
|
|||
|
|
|
|||
|
|
current = start
|
|||
|
|
while current <= end:
|
|||
|
|
target = current.strftime("%Y-%m-%d")
|
|||
|
|
try:
|
|||
|
|
self.run_daily_collection(target)
|
|||
|
|
except Exception as exc:
|
|||
|
|
print(f"{target} 数据采集失败: {exc}")
|
|||
|
|
current += pd.Timedelta(days=1)
|
|||
|
|
|
|||
|
|
def run_stock_collection(
|
|||
|
|
self,
|
|||
|
|
symbol: str,
|
|||
|
|
start_date: Optional[str] = None,
|
|||
|
|
end_date: Optional[str] = None,
|
|||
|
|
) -> None:
|
|||
|
|
"""
|
|||
|
|
获取单只股票在指定区间的资金流向数据
|
|||
|
|
Args:
|
|||
|
|
symbol: 数据库格式（SH600000、SZ000001 等）或 Tushare ts_code
|
|||
|
|
"""
|
|||
|
|
ts_code = symbol if "." in symbol else self.convert_db_code_to_tushare(symbol)
|
|||
|
|
|
|||
|
|
df = self.fetch_data(ts_code=ts_code, start_date=start_date, end_date=end_date)
|
|||
|
|
if df.empty:
|
|||
|
|
print("接口无返回数据")
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
result_df = self.transform_data(df)
|
|||
|
|
if result_df.empty:
|
|||
|
|
print("数据转换失败")
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
# 删除已有范围数据，避免重复
|
|||
|
|
if start_date or end_date:
|
|||
|
|
start = start_date or result_df["trade_date"].min().strftime("%Y-%m-%d")
|
|||
|
|
end = end_date or result_df["trade_date"].max().strftime("%Y-%m-%d")
|
|||
|
|
with self.engine.begin() as conn:
|
|||
|
|
delete_sql = text(
|
|||
|
|
f"""
|
|||
|
|
DELETE FROM {self.table_name}
|
|||
|
|
WHERE symbol = :symbol AND trade_date BETWEEN :start AND :end
|
|||
|
|
"""
|
|||
|
|
)
|
|||
|
|
affected = conn.execute(
|
|||
|
|
delete_sql,
|
|||
|
|
{
|
|||
|
|
"symbol": self.convert_tushare_code_to_db(ts_code),
|
|||
|
|
"start": start,
|
|||
|
|
"end": end,
|
|||
|
|
},
|
|||
|
|
)
|
|||
|
|
print(f"已删除 {symbol} {start}~{end} 旧数据 {affected.rowcount} 条")
|
|||
|
|
|
|||
|
|
self.save_dataframe(result_df)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def collect_moneyflow_ths(
|
|||
|
|
db_url: str,
|
|||
|
|
tushare_token: str,
|
|||
|
|
mode: str = "daily",
|
|||
|
|
date: Optional[str] = None,
|
|||
|
|
start_date: Optional[str] = None,
|
|||
|
|
end_date: Optional[str] = None,
|
|||
|
|
symbol: Optional[str] = None,
|
|||
|
|
):
|
|||
|
|
"""
|
|||
|
|
采集入口
|
|||
|
|
mode:
|
|||
|
|
- daily: 指定日期（默认今天）的全市场数据
|
|||
|
|
- range: 指定日期区间的全市场数据（逐日）
|
|||
|
|
- stock: 指定股票（symbol）的区间数据
|
|||
|
|
"""
|
|||
|
|
collector = MoneyflowTHSCollector(db_url, tushare_token)
|
|||
|
|
|
|||
|
|
if mode == "daily":
|
|||
|
|
collector.run_daily_collection(date)
|
|||
|
|
elif mode == "range":
|
|||
|
|
if not start_date or not end_date:
|
|||
|
|
raise ValueError("range 模式需要提供 start_date 和 end_date，格式 YYYY-MM-DD")
|
|||
|
|
collector.run_range_collection(start_date, end_date)
|
|||
|
|
elif mode == "stock":
|
|||
|
|
if not symbol:
|
|||
|
|
raise ValueError("stock 模式需要提供 symbol 参数")
|
|||
|
|
collector.run_stock_collection(symbol, start_date, end_date)
|
|||
|
|
elif mode == "full":
|
|||
|
|
collector.run_full_collection()
|
|||
|
|
else:
|
|||
|
|
raise ValueError(f"未知的采集模式: {mode}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
# DB_URL = "mysql+pymysql://root:Chlry#$.8@192.168.18.199:3306/db_gp_cj"
|
|||
|
|
DB_URL = "mysql+pymysql://fac_pattern:Chlry$%.8pattern@192.168.16.150:3306/factordb_mysql"
|
|||
|
|
TOKEN = TUSHARE_TOKEN
|
|||
|
|
|
|||
|
|
# 示例：
|
|||
|
|
# 1. 采集当日全市场数据
|
|||
|
|
# collect_moneyflow_ths(DB_URL, TOKEN, mode="daily")
|
|||
|
|
|
|||
|
|
# collect_moneyflow_ths(DB_URL, TOKEN, mode="full")
|
|||
|
|
|
|||
|
|
# 2. 采集指定日期全市场数据
|
|||
|
|
collect_moneyflow_ths(DB_URL, TOKEN, mode="daily", date="2025-11-17")
|
|||
|
|
#
|
|||
|
|
# 3. 采集日期区间全市场数据（逐日）
|
|||
|
|
# collect_moneyflow_ths(DB_URL, TOKEN, mode="range", start_date="2025-11-01", end_date="2025-11-07")
|
|||
|
|
#
|
|||
|
|
# 4. 采集单只股票区间数据
|
|||
|
|
# collect_moneyflow_ths(DB_URL, TOKEN, mode="stock", symbol="600519.SH", start_date="2025-11-01", end_date="2025-11-07")
|
|||
|
|
|
|||
|
|
# collect_moneyflow_ths(DB_URL, TOKEN, mode="daily", date="2025-11-07")
|
|||
|
|
|