Modernize dependency compatibility

Author: Boris-code

.gitignore

@@ -1,4 +1,5 @@
 files/*
+docs/superpowers/
 .DS_Store
 .idea/*
 */.idea/*
@@ -15,4 +16,4 @@ dist/
 media/
 .MWebMetaData/
 push.sh
-assets/
+assets/

README.md

@@ -112,7 +112,7 @@ FirstSpider|2021-02-09 14:55:14,620|air_spider.py|run|line:80|INFO| 无任务，
 
 ### Rapidproxy代理
 
-<!--5.3-->
+<!--6.7-->
 
 <a href="https://www.rapidproxy.io/?ref=boris " target="_blank">
 
@@ -133,7 +133,7 @@ FirstSpider|2021-02-09 14:55:14,620|air_spider.py|run|line:80|INFO| 无任务，
 ### NovProxy
 
 <!--5.24-->
-
+g
 <a href="https://novproxy.com?kwd=tt-git" target="_blank">
 
 <img src="https://markdown-media.oss-cn-beijing.aliyuncs.com/2026/04/24/mmexport1777023699277.jpg">

feapder/db/redisdb.py

@@ -6,11 +6,11 @@
 ---------
 @author: Boris
 """
-import os
 import time
 from typing import Union, List
 
 import redis
+from redis.cluster import ClusterNode, RedisCluster
 from redis.connection import Encoder as _Encoder
 from redis.exceptions import ConnectionError, TimeoutError
 from redis.exceptions import DataError
@@ -140,29 +140,24 @@ def get_connect(self):
                     startup_nodes = []
                     for ip_port in ip_ports:
                         ip, port = ip_port.split(":")
-                        startup_nodes.append({"host": ip, "port": port})
+                        startup_nodes.append(ClusterNode(ip, int(port)))
 
                     if self._service_name:
                         # log.debug("使用redis哨兵模式")
-                        hosts = [(node["host"], node["port"]) for node in startup_nodes]
+                        hosts = [(node.host, node.port) for node in startup_nodes]
                         sentinel = Sentinel(hosts, socket_timeout=3, **self._kwargs)
                         self._redis = sentinel.master_for(
                             self._service_name,
                             password=self._user_pass,
                             db=self._db,
-                            redis_class=redis.StrictRedis,
+                            redis_class=redis.Redis,
                             decode_responses=self._decode_responses,
                             max_connections=self._max_connections,
                             **self._kwargs,
                         )
+                        self._is_redis_cluster = False
 
                     else:
-                        try:
-                            from rediscluster import RedisCluster
-                        except ModuleNotFoundError as e:
-                            log.error('请安装 pip install "feapder[all]"')
-                            os._exit(0)
-
                         # log.debug("使用redis集群模式")
                         self._redis = RedisCluster(
                             startup_nodes=startup_nodes,
@@ -171,11 +166,10 @@ def get_connect(self):
                             max_connections=self._max_connections,
                             **self._kwargs,
                         )
-
-                    self._is_redis_cluster = True
+                        self._is_redis_cluster = True
                 else:
                     ip, port = ip_ports[0].split(":")
-                    self._redis = redis.StrictRedis(
+                    self._redis = redis.Redis(
                         host=ip,
                         port=port,
                         db=self._db,
@@ -186,7 +180,7 @@ def get_connect(self):
                     )
                     self._is_redis_cluster = False
             else:
-                self._redis = redis.StrictRedis.from_url(
+                self._redis = redis.Redis.from_url(
                     self._url, decode_responses=self._decode_responses, **self._kwargs
                 )
                 self._is_redis_cluster = False
@@ -573,7 +567,8 @@ def zexists(self, table, values):
 
         if isinstance(values, list):
             pipe = self._redis.pipeline()
-            pipe.multi()
+            if not self._is_redis_cluster:
+                pipe.multi()
             for value in values:
                 pipe.zscore(table, value)
             is_exists_temp = pipe.execute()
@@ -773,7 +768,8 @@ def setbit(
             else:
                 assert len(offsets) == len(values), "offsets值要与values值一一对应"
                 pipe = self._redis.pipeline()
-                pipe.multi()
+                if not self._is_redis_cluster:
+                    pipe.multi()
 
                 for offset, value in zip(offsets, values):
                     pipe.setbit(table, offset, value)
@@ -792,7 +788,8 @@ def getbit(self, table, offsets):
         """
         if isinstance(offsets, list):
             pipe = self._redis.pipeline()
-            pipe.multi()
+            if not self._is_redis_cluster:
+                pipe.multi()
             for offset in offsets:
                 pipe.getbit(table, offset)
 

feapder/network/proxy_pool_old.py

@@ -147,7 +147,7 @@ def get_proxy_from_redis(proxy_source_url, **kwargs):
     @return: [{'http':'http://xxx.xxx.xxx:xxx', 'https':'http://xxx.xxx.xxx.xxx:xxx'}]
     """
 
-    redis_conn = redis.StrictRedis.from_url(proxy_source_url)
+    redis_conn = redis.Redis.from_url(proxy_source_url)
     key = kwargs.get("redis_proxies_key")
     assert key, "从redis中获取代理 需要指定 redis_proxies_key"
     proxies = redis_conn.zrange(key, 0, -1)

feapder/requirements.txt

@@ -1,21 +1,20 @@
 better-exceptions>=0.2.2
-DBUtils>=2.0
-parsel>=1.5.2
+DBUtils>=3.0
+parsel>=1.8.1
 PyExecJS>=1.5.1
-pymongo>=3.10.1
-PyMySQL>=0.9.3
-redis>=2.10.6,<4.0.0
-requests>=2.22.0
-selenium>=3.141.0
-bs4>=0.0.1
-ipython>=7.14.0
-bitarray>=1.5.3
-redis-py-cluster>=2.1.0
-cryptography>=3.3.2
-urllib3>=1.25.8
+pymongo>=4.0.0
+PyMySQL>=1.1.0
+redis>=5.0.0,<9.0.0
+requests>=2.31.0
+selenium>=4.10.0
+beautifulsoup4>=4.12.0
+ipython>=8.0.0
+bitarray>=2.8.0
+cryptography>=41.0.0
+urllib3>=2.0.0,<3.0.0
 loguru>=0.5.3
 influxdb>=5.3.1
 pyperclip>=1.8.2
 webdriver-manager>=4.0.0
 terminal-layout>=2.1.3
-playwright
+playwright>=1.40.0