爬虫支持重复启动

Boris-code · Boris-code · commit 3108880eeb38 · 2021-05-23T15:35:55.000+08:00
diff --git a/feapder/buffer/item_buffer.py b/feapder/buffer/item_buffer.py
@@ -27,15 +27,7 @@
 MYSQL_PIPELINE_PATH = "feapder.pipelines.mysql_pipeline.MysqlPipeline"
 
 
-class Singleton(object):
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, "_inst"):
-            cls._inst = super(Singleton, cls).__new__(cls)
-
-        return cls._inst
-
-
-class ItemBuffer(threading.Thread, Singleton):
+class ItemBuffer(threading.Thread):
     dedup = None
     __redis_db = None
 
diff --git a/feapder/buffer/request_buffer.py b/feapder/buffer/request_buffer.py
@@ -14,21 +14,13 @@
 import feapder.setting as setting
 import feapder.utils.tools as tools
 from feapder.db.redisdb import RedisDB
-from feapder.utils.log import log
 from feapder.dedup import Dedup
+from feapder.utils.log import log
 
 MAX_URL_COUNT = 1000  # 缓存中最大request数
 
 
-class Singleton(object):
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, "_inst"):
-            cls._inst = super(Singleton, cls).__new__(cls)
-
-        return cls._inst
-
-
-class RequestBuffer(threading.Thread, Singleton):
+class RequestBuffer(threading.Thread):
     dedup = None
 
     def __init__(self, redis_key):
diff --git a/feapder/core/spiders/batch_spider.py b/feapder/core/spiders/batch_spider.py
@@ -17,7 +17,6 @@
 import feapder.setting as setting
 import feapder.utils.tools as tools
 from feapder.buffer.item_buffer import MAX_ITEM_COUNT
-from feapder.buffer.request_buffer import RequestBuffer
 from feapder.core.base_parser import BatchParser
 from feapder.core.scheduler import Scheduler
 from feapder.db.mysqldb import MysqlDB
@@ -118,8 +117,6 @@ def __init__(
         self._redisdb = RedisDB()
         self._mysqldb = MysqlDB()
 
-        self._request_buffer = RequestBuffer(self._redis_key)
-
         self._task_table = task_table  # mysql中的任务表
         self._batch_record_table = batch_record_table  # mysql 中的批次记录表
         self._batch_name = batch_name  # 批次采集程序名称
@@ -1072,7 +1069,7 @@ def __init__(
         save_to_db=False,
         update_stask=False,
         *args,
-        **kwargs
+        **kwargs,
     ):
         """
         @param task_id:  任务id
