342 lines
12 KiB
Python
342 lines
12 KiB
Python
|
import requests
|
|||
|
|
import redis
|
|||
|
|
import random
|
|||
|
|
import time
|
|||
|
|
import threading
|
|||
|
|
import json
|
|||
|
|
from typing import Dict, List, Optional, Union
|
|||
|
|
from datetime import datetime, timedelta
|
|||
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|||
|
|
|
|||
|
|
|
|||
|
|
class EnhancedProxyManager:
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
proxy_api_url: str = 'https://dps.kdlapi.com/api/getdps?secret_id=o4itop21b4byqg47eevx&signature=3d3fuvm6raah1xyjecl6bby1mj6gtx0c&num=3&format=json',
|
|||
|
|
valid_check_url: str = 'https://dps.kdlapi.com/api/checkdpsvalid?secret_id=o4itop21b4byqg47eevx&signature=3d3fuvm6raah1xyjecl6bby1mj6gtx0c',
|
|||
|
|
redis_host: str = '192.168.18.123',
|
|||
|
|
redis_port: int = 6379,
|
|||
|
|
redis_db: int = 7,
|
|||
|
|
redis_password: str = 'wlkj2018',
|
|||
|
|
redis_key: str = 'proxy_pool',
|
|||
|
|
update_interval: int = 3600,
|
|||
|
|
max_retries: int = 3,
|
|||
|
|
proxy_timeout: int = 10,
|
|||
|
|
auto_refresh: bool = False,
|
|||
|
|
max_pool_size: int = 50,
|
|||
|
|
enable_api_validation: bool = True # 新增：是否启用API验证开关
|
|||
|
|
):
|
|||
|
|
"""
|
|||
|
|
增强版代理管理器 - 支持多IP池和手动代理管理
|
|||
|
|
|
|||
|
|
:param proxy_api_url: 获取代理的API地址
|
|||
|
|
:param redis_host: Redis主机地址
|
|||
|
|
:param redis_port: Redis端口
|
|||
|
|
:param redis_db: Redis数据库
|
|||
|
|
:param redis_password: Redis密码
|
|||
|
|
:param redis_key: Redis中存储代理的键前缀
|
|||
|
|
:param update_interval: 代理更新间隔(秒)
|
|||
|
|
:param max_retries: 最大重试次数
|
|||
|
|
:param proxy_timeout: 代理连接超时时间(秒)
|
|||
|
|
:param auto_refresh: 是否自动从API获取代理
|
|||
|
|
:param max_pool_size: 代理池最大容量
|
|||
|
|
"""
|
|||
|
|
self.proxy_api_url = proxy_api_url
|
|||
|
|
self.redis_key_prefix = redis_key
|
|||
|
|
self.update_interval = update_interval
|
|||
|
|
self.max_retries = max_retries
|
|||
|
|
self.proxy_timeout = proxy_timeout
|
|||
|
|
self.auto_refresh = auto_refresh
|
|||
|
|
self.max_pool_size = max_pool_size
|
|||
|
|
self.valid_check_url = valid_check_url
|
|||
|
|
self.enable_api_validation = enable_api_validation
|
|||
|
|
|
|||
|
|
# Redis连接
|
|||
|
|
self.redis_conn = redis.StrictRedis(
|
|||
|
|
host=redis_host,
|
|||
|
|
port=redis_port,
|
|||
|
|
db=redis_db,
|
|||
|
|
password=redis_password,
|
|||
|
|
decode_responses=True
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# 线程安全控制
|
|||
|
|
self.lock = threading.Lock()
|
|||
|
|
self.condition = threading.Condition()
|
|||
|
|
|
|||
|
|
# 启动维护线程
|
|||
|
|
if self.auto_refresh:
|
|||
|
|
self._start_maintenance_thread()
|
|||
|
|
|
|||
|
|
def _start_maintenance_thread(self):
|
|||
|
|
"""启动后台维护线程"""
|
|||
|
|
|
|||
|
|
def maintenance_loop():
|
|||
|
|
while True:
|
|||
|
|
with self.condition:
|
|||
|
|
self._refresh_api_proxies()
|
|||
|
|
self.condition.notify_all()
|
|||
|
|
time.sleep(self.update_interval)
|
|||
|
|
|
|||
|
|
thread = threading.Thread(target=maintenance_loop, daemon=True)
|
|||
|
|
thread.start()
|
|||
|
|
|
|||
|
|
def _get_redis_key(self, proxy_type: str) -> str:
|
|||
|
|
"""获取Redis键名"""
|
|||
|
|
return f"{self.redis_key_prefix}:{proxy_type}"
|
|||
|
|
|
|||
|
|
def _check_proxy_valid(self, proxy_list: List[str]) -> Dict[str, bool]:
|
|||
|
|
"""通过API检查代理是否有效"""
|
|||
|
|
if not self.enable_api_validation or not proxy_list:
|
|||
|
|
return {}
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 拼接代理参数：proxy=ip1:port1,ip2:port2
|
|||
|
|
proxy_param = '&proxy=' + ','.join(proxy_list)
|
|||
|
|
response = requests.get(self.valid_check_url + proxy_param, timeout=10)
|
|||
|
|
|
|||
|
|
if response.status_code == 200:
|
|||
|
|
data = response.json()
|
|||
|
|
if data.get('code') == 0:
|
|||
|
|
return data.get('data', {})
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"API验证代理有效性失败: {e}")
|
|||
|
|
return {}
|
|||
|
|
|
|||
|
|
def _refresh_api_proxies(self) -> bool:
|
|||
|
|
"""从API获取最新代理并存入Redis"""
|
|||
|
|
api_key = self._get_redis_key('api')
|
|||
|
|
|
|||
|
|
# 前置检查：如果已有足够数量且未过期的代理，则不需要刷新
|
|||
|
|
existing_proxies = self.redis_conn.hgetall(api_key)
|
|||
|
|
|
|||
|
|
# 检查代理数量是否足够（大于等于4）且未过期
|
|||
|
|
if len(existing_proxies) >= 4:
|
|||
|
|
# 检查代理是否过期（假设代理有效期为24小时）
|
|||
|
|
current_time = datetime.now()
|
|||
|
|
need_refresh = False
|
|||
|
|
|
|||
|
|
for proxy_json in existing_proxies.values():
|
|||
|
|
proxy = json.loads(proxy_json)
|
|||
|
|
last_checked = datetime.strptime(proxy['last_checked'], "%Y-%m-%d %H:%M:%S")
|
|||
|
|
if (current_time - last_checked) > timedelta(hours=1):
|
|||
|
|
need_refresh = True
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
if not need_refresh:
|
|||
|
|
print("当前有足够数量且未过期的代理，无需刷新")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
response = requests.get(self.proxy_api_url, timeout=self.proxy_timeout)
|
|||
|
|
if response.status_code == 200:
|
|||
|
|
data = response.json()
|
|||
|
|
if data.get('code') == 0 and data.get('data'):
|
|||
|
|
# 清空旧API代理
|
|||
|
|
api_key = self._get_redis_key('api')
|
|||
|
|
# self.redis_conn.delete(api_key)
|
|||
|
|
|
|||
|
|
# 添加新代理
|
|||
|
|
for proxy_data in data['data']['proxy_list']:
|
|||
|
|
proxy = {
|
|||
|
|
'http': f"http://{proxy_data}",
|
|||
|
|
'https': f"http://{proxy_data}",
|
|||
|
|
'server': proxy_data,
|
|||
|
|
'source': 'api',
|
|||
|
|
'last_checked': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
|||
|
|
'status': 'active'
|
|||
|
|
}
|
|||
|
|
self.redis_conn.hset(
|
|||
|
|
api_key,
|
|||
|
|
proxy_data,
|
|||
|
|
json.dumps(proxy)
|
|||
|
|
)
|
|||
|
|
return True
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"更新API代理失败: {e}")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
def add_manual_proxies(self, proxies: Union[List[str], str]) -> int:
|
|||
|
|
"""
|
|||
|
|
手动添加代理到池中
|
|||
|
|
|
|||
|
|
:param proxies: 代理列表(格式: ["ip:port", ...] 或 "ip:port")
|
|||
|
|
:return: 成功添加的代理数量
|
|||
|
|
"""
|
|||
|
|
if isinstance(proxies, str):
|
|||
|
|
proxies = [proxies]
|
|||
|
|
|
|||
|
|
added = 0
|
|||
|
|
manual_key = self._get_redis_key('manual')
|
|||
|
|
|
|||
|
|
# 验证并添加代理
|
|||
|
|
def _check_and_add(proxy):
|
|||
|
|
nonlocal added
|
|||
|
|
proxy_dict = {
|
|||
|
|
'http': f"http://{proxy}",
|
|||
|
|
'https': f"http://{proxy}",
|
|||
|
|
'server': proxy,
|
|||
|
|
'source': 'manual',
|
|||
|
|
'last_checked': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
|||
|
|
'status': 'active'
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if self._is_proxy_working(proxy_dict):
|
|||
|
|
with self.lock:
|
|||
|
|
# 检查是否已存在
|
|||
|
|
if not self.redis_conn.hexists(manual_key, proxy):
|
|||
|
|
self.redis_conn.hset(manual_key, proxy, json.dumps(proxy_dict))
|
|||
|
|
added += 1
|
|||
|
|
|
|||
|
|
# 使用线程池并行验证
|
|||
|
|
with ThreadPoolExecutor(max_workers=10) as executor:
|
|||
|
|
executor.map(_check_and_add, proxies)
|
|||
|
|
|
|||
|
|
print(f"成功添加 {added} 个手动代理")
|
|||
|
|
return added
|
|||
|
|
|
|||
|
|
def _is_proxy_working(self, proxy: Dict) -> bool:
|
|||
|
|
"""检查代理是否可用"""
|
|||
|
|
test_urls = [
|
|||
|
|
"http://httpbin.org/ip",
|
|||
|
|
"http://www.google.com/gen_204"
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
proxies = {
|
|||
|
|
'http': proxy['http'],
|
|||
|
|
'https': proxy['https']
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for url in test_urls:
|
|||
|
|
try:
|
|||
|
|
response = requests.get(
|
|||
|
|
url,
|
|||
|
|
proxies=proxies,
|
|||
|
|
timeout=self.proxy_timeout,
|
|||
|
|
allow_redirects=False
|
|||
|
|
)
|
|||
|
|
if 200 <= response.status_code < 500:
|
|||
|
|
return True
|
|||
|
|
except:
|
|||
|
|
continue
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
def get_random_proxy(self) -> Optional[Dict]:
|
|||
|
|
"""
|
|||
|
|
随机获取一个可用代理
|
|||
|
|
|
|||
|
|
:return: 代理字典或None(如果无可用代理)
|
|||
|
|
"""
|
|||
|
|
# 优先从API代理获取
|
|||
|
|
# api_key = self._get_redis_key('api')
|
|||
|
|
manual_key = self._get_redis_key('manual')
|
|||
|
|
|
|||
|
|
# 获取所有活跃代理
|
|||
|
|
proxies = []
|
|||
|
|
|
|||
|
|
# 先检查API代理
|
|||
|
|
# api_proxies = self.redis_conn.hgetall(api_key)
|
|||
|
|
# for proxy_json in api_proxies.values():
|
|||
|
|
# proxy = json.loads(proxy_json)
|
|||
|
|
# if proxy.get('status') == 'active':
|
|||
|
|
# proxies.append(proxy)
|
|||
|
|
|
|||
|
|
# 如果API代理不可用或auto_refresh关闭，检查手动代理
|
|||
|
|
if not proxies or not self.auto_refresh:
|
|||
|
|
manual_proxies = self.redis_conn.hgetall(manual_key)
|
|||
|
|
for proxy_json in manual_proxies.values():
|
|||
|
|
proxy = json.loads(proxy_json)
|
|||
|
|
if proxy.get('status') == 'active':
|
|||
|
|
proxies.append(proxy)
|
|||
|
|
|
|||
|
|
if not proxies:
|
|||
|
|
if self.auto_refresh:
|
|||
|
|
print("代理池为空，尝试从API获取...")
|
|||
|
|
self._refresh_api_proxies()
|
|||
|
|
return self.get_random_proxy()
|
|||
|
|
else:
|
|||
|
|
print("代理池为空且自动刷新已关闭")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
# 随机选择一个代理
|
|||
|
|
selected = random.choice(proxies)
|
|||
|
|
selected['_redis_key'] = self._get_redis_key(selected['source'])
|
|||
|
|
return selected
|
|||
|
|
|
|||
|
|
def mark_proxy_failed(self, proxy: Dict):
|
|||
|
|
"""标记代理为失败并从池中移除"""
|
|||
|
|
if '_redis_key' not in proxy:
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
# 如果是API代理且启用验证，先检查是否真的失效
|
|||
|
|
if proxy.get('source') == 'api' and self.enable_api_validation:
|
|||
|
|
valid_status = self._check_proxy_valid([proxy['server']])
|
|||
|
|
is_valid = valid_status.get(proxy['server'], False)
|
|||
|
|
|
|||
|
|
if is_valid:
|
|||
|
|
print(f"代理 {proxy['server']} API验证仍有效，暂不移除")
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
with self.lock:
|
|||
|
|
self.redis_conn.hdel(proxy['_redis_key'], proxy['server'])
|
|||
|
|
print(f"代理 {proxy['server']} 已被移除")
|
|||
|
|
|
|||
|
|
def request_with_proxy(
|
|||
|
|
self,
|
|||
|
|
method: str,
|
|||
|
|
url: str,
|
|||
|
|
retry_count: int = 0,
|
|||
|
|
**kwargs
|
|||
|
|
) -> requests.Response:
|
|||
|
|
"""
|
|||
|
|
使用代理发送请求
|
|||
|
|
|
|||
|
|
:param method: HTTP方法
|
|||
|
|
:param url: 请求URL
|
|||
|
|
:param retry_count: 内部重试计数
|
|||
|
|
:param kwargs: 其他requests参数
|
|||
|
|
:return: Response对象
|
|||
|
|
"""
|
|||
|
|
if retry_count >= self.max_retries:
|
|||
|
|
raise requests.exceptions.RequestException(f"达到最大重试次数 {self.max_retries}")
|
|||
|
|
|
|||
|
|
proxy = self.get_random_proxy()
|
|||
|
|
if not proxy:
|
|||
|
|
raise requests.exceptions.RequestException("无可用代理")
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
response = requests.request(
|
|||
|
|
method,
|
|||
|
|
url,
|
|||
|
|
proxies={
|
|||
|
|
'http': proxy['http'],
|
|||
|
|
'https': proxy['https']
|
|||
|
|
},
|
|||
|
|
timeout=self.proxy_timeout,
|
|||
|
|
**kwargs
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if response.status_code >= 400:
|
|||
|
|
raise requests.exceptions.HTTPError(
|
|||
|
|
f"HTTP错误: {response.status_code}",
|
|||
|
|
response=response
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return response
|
|||
|
|
|
|||
|
|
except requests.exceptions.RequestException as e:
|
|||
|
|
print(f"代理 {proxy['server']} 请求失败: {e}")
|
|||
|
|
self.mark_proxy_failed(proxy)
|
|||
|
|
return self.request_with_proxy(method, url, retry_count + 1, **kwargs)
|
|||
|
|
|
|||
|
|
def get_pool_status(self) -> Dict:
|
|||
|
|
"""获取代理池状态"""
|
|||
|
|
api_key = self._get_redis_key('api')
|
|||
|
|
manual_key = self._get_redis_key('manual')
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
'api_proxies': self.redis_conn.hlen(api_key),
|
|||
|
|
'manual_proxies': self.redis_conn.hlen(manual_key),
|
|||
|
|
'auto_refresh': self.auto_refresh,
|
|||
|
|
'last_update': datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|||
|
|
}
|