2025-04-02 13:52:34 +08:00
|
|
|
|
# coding:utf-8
|
|
|
|
|
|
|
|
|
|
import requests
|
|
|
|
|
import pandas as pd
|
|
|
|
|
from sqlalchemy import create_engine, text
|
2025-06-24 10:51:05 +08:00
|
|
|
|
from datetime import datetime, timedelta
|
2025-04-02 13:52:34 +08:00
|
|
|
|
from tqdm import tqdm
|
2025-05-19 17:02:52 +08:00
|
|
|
|
from src.scripts.config import XUEQIU_HEADERS
|
2025-05-14 08:54:56 +08:00
|
|
|
|
import gc
|
2025-04-02 13:52:34 +08:00
|
|
|
|
|
|
|
|
|
class StockDailyDataCollector:
|
|
|
|
|
"""股票日线数据采集器类"""
|
2025-05-14 08:54:56 +08:00
|
|
|
|
|
2025-04-02 13:52:34 +08:00
|
|
|
|
def __init__(self, db_url):
|
2025-05-14 08:54:56 +08:00
|
|
|
|
self.engine = create_engine(
|
|
|
|
|
db_url,
|
|
|
|
|
pool_size=5,
|
|
|
|
|
max_overflow=10,
|
|
|
|
|
pool_recycle=3600
|
|
|
|
|
)
|
2025-04-02 13:52:34 +08:00
|
|
|
|
self.headers = XUEQIU_HEADERS
|
|
|
|
|
|
|
|
|
|
def fetch_all_stock_codes(self):
|
2025-05-19 17:02:52 +08:00
|
|
|
|
# 从gp_code_all获取股票代码
|
|
|
|
|
query_all = "SELECT gp_code FROM gp_code_all"
|
|
|
|
|
df_all = pd.read_sql(query_all, self.engine)
|
|
|
|
|
codes_all = df_all['gp_code'].tolist()
|
|
|
|
|
|
|
|
|
|
# 从gp_code_zs获取股票代码
|
|
|
|
|
query_zs = "SELECT gp_code FROM gp_code_zs"
|
|
|
|
|
df_zs = pd.read_sql(query_zs, self.engine)
|
|
|
|
|
codes_zs = df_zs['gp_code'].tolist()
|
2025-06-24 10:51:05 +08:00
|
|
|
|
|
|
|
|
|
# 从gp_code_hk获取股票代码
|
|
|
|
|
query_hk = "SELECT gp_code FROM gp_code_hk"
|
|
|
|
|
df_hk = pd.read_sql(query_hk, self.engine)
|
|
|
|
|
codes_hk = df_hk['gp_code'].tolist()
|
2025-05-19 17:02:52 +08:00
|
|
|
|
|
|
|
|
|
# 合并去重
|
2025-06-24 10:51:05 +08:00
|
|
|
|
all_codes = list(set(codes_all + codes_zs + codes_hk))
|
|
|
|
|
print(f"获取到股票代码: {len(codes_all)}个来自gp_code_all, {len(codes_zs)}个来自gp_code_zs, {len(codes_hk)}个来自gp_code_hk, 去重后共{len(all_codes)}个")
|
2025-05-19 17:02:52 +08:00
|
|
|
|
return all_codes
|
2025-04-02 13:52:34 +08:00
|
|
|
|
|
2025-06-24 10:51:05 +08:00
|
|
|
|
def fetch_daily_stock_data(self, symbol, begin, count=-1):
|
|
|
|
|
"""获取日线数据,count=-1表示最新一天,-2表示最近两天,-1800表示最近1800天"""
|
|
|
|
|
url = f"https://stock.xueqiu.com/v5/stock/chart/kline.json?symbol={symbol}&begin={begin}&period=day&type=before&count={count}&indicator=kline,pe,pb,ps,pcf,market_capital,agt,ggt,balance"
|
2025-05-14 08:54:56 +08:00
|
|
|
|
try:
|
2025-06-24 10:51:05 +08:00
|
|
|
|
response = requests.get(url, headers=self.headers, timeout=20)
|
2025-05-14 08:54:56 +08:00
|
|
|
|
return response.json()
|
|
|
|
|
except Exception as e:
|
|
|
|
|
print(f"Request error for {symbol}: {e}")
|
|
|
|
|
return {'error_code': -1, 'error_description': str(e)}
|
|
|
|
|
|
|
|
|
|
def transform_data(self, data, symbol):
|
2025-04-02 13:52:34 +08:00
|
|
|
|
try:
|
|
|
|
|
items = data['data']['item']
|
|
|
|
|
columns = data['data']['column']
|
|
|
|
|
except KeyError as e:
|
|
|
|
|
print(f"KeyError for {symbol}: {e}")
|
2025-05-14 08:54:56 +08:00
|
|
|
|
return None
|
2025-04-02 13:52:34 +08:00
|
|
|
|
|
|
|
|
|
df = pd.DataFrame(items, columns=columns)
|
|
|
|
|
df['symbol'] = symbol
|
2025-05-14 08:54:56 +08:00
|
|
|
|
|
|
|
|
|
required_columns = ['timestamp', 'volume', 'open', 'high', 'low', 'close',
|
|
|
|
|
'chg', 'percent', 'turnoverrate', 'amount', 'symbol', 'pb', 'pe', 'ps']
|
2025-04-02 13:52:34 +08:00
|
|
|
|
existing_columns = [col for col in required_columns if col in df.columns]
|
|
|
|
|
df = df[existing_columns]
|
2025-05-14 08:54:56 +08:00
|
|
|
|
|
2025-04-02 13:52:34 +08:00
|
|
|
|
if 'timestamp' in df.columns:
|
|
|
|
|
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True).dt.tz_convert('Asia/Shanghai')
|
2025-05-14 08:54:56 +08:00
|
|
|
|
|
|
|
|
|
return df
|
|
|
|
|
|
|
|
|
|
def save_batch_to_database(self, batch):
|
|
|
|
|
if batch:
|
|
|
|
|
df_all = pd.concat(batch, ignore_index=True)
|
|
|
|
|
df_all.to_sql('gp_day_data', self.engine, if_exists='append', index=False)
|
2025-04-02 13:52:34 +08:00
|
|
|
|
|
|
|
|
|
def fetch_data_for_date(self, date=None):
|
|
|
|
|
if date is None:
|
|
|
|
|
start_date = datetime.now()
|
|
|
|
|
date_str = start_date.strftime('%Y-%m-%d')
|
|
|
|
|
else:
|
|
|
|
|
start_date = datetime.strptime(date, '%Y-%m-%d')
|
|
|
|
|
date_str = date
|
|
|
|
|
|
2025-05-14 08:54:56 +08:00
|
|
|
|
delete_query = text("DELETE FROM gp_day_data WHERE `timestamp` LIKE :date_str")
|
|
|
|
|
with self.engine.begin() as conn:
|
2025-04-02 13:52:34 +08:00
|
|
|
|
conn.execute(delete_query, {"date_str": f"{date_str}%"})
|
2025-05-14 08:54:56 +08:00
|
|
|
|
|
2025-04-02 13:52:34 +08:00
|
|
|
|
stock_codes = self.fetch_all_stock_codes()
|
2025-05-14 08:54:56 +08:00
|
|
|
|
begin = int(start_date.replace(hour=0, minute=0, second=0, microsecond=0).timestamp() * 1000)
|
2025-04-02 13:52:34 +08:00
|
|
|
|
|
2025-05-14 08:54:56 +08:00
|
|
|
|
batch_data = []
|
|
|
|
|
for idx, symbol in enumerate(tqdm(stock_codes, desc=f"Fetching and saving daily stock data for {date_str}")):
|
2025-04-02 13:52:34 +08:00
|
|
|
|
data = self.fetch_daily_stock_data(symbol, begin)
|
2025-05-14 08:54:56 +08:00
|
|
|
|
|
|
|
|
|
if data.get('error_code') == 0:
|
|
|
|
|
df = self.transform_data(data, symbol)
|
|
|
|
|
if df is not None:
|
|
|
|
|
batch_data.append(df)
|
2025-04-02 13:52:34 +08:00
|
|
|
|
else:
|
2025-05-14 08:54:56 +08:00
|
|
|
|
print(f"Error fetching data for {symbol} on {date_str}: {data.get('error_description')}")
|
|
|
|
|
|
|
|
|
|
if len(batch_data) >= 100:
|
|
|
|
|
self.save_batch_to_database(batch_data)
|
|
|
|
|
batch_data.clear()
|
|
|
|
|
gc.collect()
|
|
|
|
|
|
|
|
|
|
# Save remaining data
|
|
|
|
|
if batch_data:
|
|
|
|
|
self.save_batch_to_database(batch_data)
|
|
|
|
|
gc.collect()
|
2025-04-02 13:52:34 +08:00
|
|
|
|
|
2025-05-14 08:54:56 +08:00
|
|
|
|
self.engine.dispose()
|
2025-04-02 13:52:34 +08:00
|
|
|
|
print(f"Daily data fetching and saving completed for {date_str}.")
|
|
|
|
|
|
2025-06-24 10:51:05 +08:00
|
|
|
|
def delete_stock_history(self, symbol):
|
|
|
|
|
"""删除指定股票的全部历史数据"""
|
|
|
|
|
delete_query = text("DELETE FROM gp_day_data WHERE symbol = :symbol")
|
|
|
|
|
try:
|
|
|
|
|
with self.engine.begin() as conn:
|
|
|
|
|
conn.execute(delete_query, {"symbol": symbol})
|
|
|
|
|
print(f"Deleted history for {symbol}")
|
|
|
|
|
return True
|
|
|
|
|
except Exception as e:
|
|
|
|
|
print(f"Error deleting history for {symbol}: {e}")
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
def refetch_and_save_history(self, symbol, days=1800):
|
|
|
|
|
"""重新获取并保存指定股票的长期历史数据"""
|
|
|
|
|
print(f"Refetching last {days} days for {symbol}...")
|
|
|
|
|
begin = int(datetime.now().timestamp() * 1000)
|
|
|
|
|
data = self.fetch_daily_stock_data(symbol, begin, count=-days)
|
|
|
|
|
if data.get('error_code') == 0:
|
|
|
|
|
df = self.transform_data(data, symbol)
|
|
|
|
|
if df is not None and not df.empty:
|
|
|
|
|
self.save_batch_to_database([df])
|
|
|
|
|
print(f"Successfully refetched and saved history for {symbol}.")
|
|
|
|
|
else:
|
|
|
|
|
print(f"No data transformed for {symbol} after refetch.")
|
|
|
|
|
else:
|
|
|
|
|
print(f"Error refetching history for {symbol}: {data.get('error_description')}")
|
|
|
|
|
|
|
|
|
|
def check_and_fix_ex_rights_data(self):
|
|
|
|
|
"""
|
|
|
|
|
检查所有股票是否发生除权,如果发生,则删除历史数据并重新获取。
|
|
|
|
|
新逻辑:直接用API返回的上个交易日的时间戳去数据库查询,更稳妥。
|
|
|
|
|
记录:除权日期、股票代码()、
|
|
|
|
|
"""
|
|
|
|
|
all_codes = self.fetch_all_stock_codes()
|
|
|
|
|
ex_rights_log_data = []
|
|
|
|
|
|
|
|
|
|
print("--- Step 1: Checking for ex-rights stocks ---")
|
|
|
|
|
for symbol in tqdm(all_codes, desc="Comparing prices"):
|
|
|
|
|
# 1. 从API获取最近两天的日线数据
|
|
|
|
|
begin = int(datetime.now().timestamp() * 1000)
|
|
|
|
|
data = self.fetch_daily_stock_data(symbol, begin, count=-2)
|
|
|
|
|
|
|
|
|
|
api_timestamp_str = None
|
|
|
|
|
api_close = None
|
|
|
|
|
|
|
|
|
|
if data.get('error_code') == 0 and data.get('data', {}).get('item') and len(data['data']['item']) >= 2:
|
|
|
|
|
try:
|
|
|
|
|
# API返回的数据是按时间升序的,[-2]是上个交易日
|
|
|
|
|
prev_day_data = data['data']['item'][-2]
|
|
|
|
|
columns = data['data']['column']
|
|
|
|
|
|
|
|
|
|
timestamp_index = columns.index('timestamp')
|
|
|
|
|
close_index = columns.index('close')
|
|
|
|
|
|
|
|
|
|
api_timestamp_ms = prev_day_data[timestamp_index]
|
|
|
|
|
api_close = prev_day_data[close_index]
|
|
|
|
|
|
|
|
|
|
# 将毫秒时间戳转换为'YYYY-MM-DD'格式,用于数据库查询
|
|
|
|
|
api_timestamp_str = pd.to_datetime(api_timestamp_ms, unit='ms', utc=True).tz_convert('Asia/Shanghai').strftime('%Y-%m-%d')
|
|
|
|
|
except (ValueError, IndexError, TypeError) as e:
|
|
|
|
|
print(f"\nError parsing API data for {symbol}: {e}")
|
|
|
|
|
continue # 处理下一只股票
|
|
|
|
|
else:
|
|
|
|
|
# 获取API数据失败或数据不足,跳过此股票
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# 如果未能从API解析出上个交易日的数据,则跳过
|
|
|
|
|
if api_timestamp_str is None or api_close is None:
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# 2. 根据API返回的时间戳,从数据库查询当天的收盘价
|
|
|
|
|
db_close = None
|
|
|
|
|
query = text("SELECT `close` FROM gp_day_data WHERE symbol = :symbol AND `timestamp` LIKE :date_str")
|
|
|
|
|
try:
|
|
|
|
|
with self.engine.connect() as conn:
|
|
|
|
|
result = conn.execute(query, {"symbol": symbol, "date_str": f"{api_timestamp_str}%"}).fetchone()
|
|
|
|
|
db_close = result[0] if result else None
|
|
|
|
|
except Exception as e:
|
|
|
|
|
print(f"\nError getting DB close for {symbol} on {api_timestamp_str}: {e}")
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# 3. 比较价格
|
|
|
|
|
if db_close is not None:
|
|
|
|
|
# 注意:数据库中取出的db_close可能是Decimal类型,需要转换
|
|
|
|
|
if not abs(float(db_close) - api_close) < 0.001:
|
|
|
|
|
print(f"\nEx-rights detected for {symbol} on {api_timestamp_str}: DB_close={db_close}, API_close={api_close}")
|
|
|
|
|
ex_rights_log_data.append({
|
|
|
|
|
'symbol': symbol,
|
|
|
|
|
'date': datetime.now().strftime('%Y-%m-%d'),
|
|
|
|
|
'db_price': float(db_close),
|
|
|
|
|
'api_price': api_close,
|
|
|
|
|
'log_time': datetime.now()
|
|
|
|
|
})
|
|
|
|
|
# 如果数据库当天没有数据,我们无法比较,所以不处理。
|
|
|
|
|
# 这可能是新股或之前采集失败,不属于除权范畴。
|
|
|
|
|
|
|
|
|
|
# 4. 对发生除权的股票进行记录和修复
|
|
|
|
|
if not ex_rights_log_data:
|
|
|
|
|
print("\n--- No ex-rights stocks found. Data is consistent. ---")
|
|
|
|
|
self.engine.dispose()
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
# 在修复前,先将日志保存到数据库
|
|
|
|
|
self.save_ex_rights_log(ex_rights_log_data)
|
|
|
|
|
|
|
|
|
|
# 从日志数据中提取出需要修复的股票代码列表
|
|
|
|
|
ex_rights_stocks = [item['symbol'] for item in ex_rights_log_data]
|
|
|
|
|
|
|
|
|
|
print(f"\n--- Step 2: Found {len(ex_rights_stocks)} stocks to fix: {ex_rights_stocks} ---")
|
|
|
|
|
for symbol in tqdm(ex_rights_stocks, desc="Fixing data"):
|
|
|
|
|
if self.delete_stock_history(symbol):
|
|
|
|
|
self.refetch_and_save_history(symbol, days=1800)
|
|
|
|
|
|
|
|
|
|
self.engine.dispose()
|
|
|
|
|
print("\n--- Ex-rights data fixing process completed. ---")
|
|
|
|
|
|
|
|
|
|
def save_ex_rights_log(self, log_data: list):
|
|
|
|
|
"""将除权日志保存到数据库"""
|
|
|
|
|
if not log_data:
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
print(f"--- Saving {len(log_data)} ex-rights events to log table... ---")
|
|
|
|
|
try:
|
|
|
|
|
df = pd.DataFrame(log_data)
|
|
|
|
|
# 确保列名与数据库字段匹配
|
|
|
|
|
df = df.rename(columns={
|
|
|
|
|
'symbol': 'stock_code',
|
|
|
|
|
'date': 'change_date',
|
|
|
|
|
'db_price': 'before_price',
|
|
|
|
|
'api_price': 'after_price',
|
|
|
|
|
'log_time': 'update_time'
|
|
|
|
|
})
|
|
|
|
|
df.to_sql('gp_ex_rights_log', self.engine, if_exists='append', index=False)
|
|
|
|
|
print("--- Ex-rights log saved successfully. ---")
|
|
|
|
|
except Exception as e:
|
|
|
|
|
print(f"!!! Error saving ex-rights log: {e}")
|
|
|
|
|
|
2025-04-02 13:52:34 +08:00
|
|
|
|
def collect_stock_daily_data(db_url, date=None):
|
|
|
|
|
collector = StockDailyDataCollector(db_url)
|
|
|
|
|
collector.fetch_data_for_date(date)
|
2025-06-24 10:51:05 +08:00
|
|
|
|
collector.check_and_fix_ex_rights_data()
|
2025-04-02 13:52:34 +08:00
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
2025-04-02 16:16:35 +08:00
|
|
|
|
db_url = 'mysql+pymysql://root:Chlry#$.8@192.168.18.199:3306/db_gp_cj'
|
2025-06-24 10:51:05 +08:00
|
|
|
|
|
|
|
|
|
# --- 使用方式 ---
|
|
|
|
|
# 1. 日常采集当天数据
|
|
|
|
|
# collect_stock_daily_data(db_url)
|
|
|
|
|
|
|
|
|
|
# 2. 手动执行除权检查和数据修复
|
|
|
|
|
# collector = StockDailyDataCollector(db_url)
|
|
|
|
|
# collector.check_and_fix_ex_rights_data()
|