队列优化，支持二次魔改

baid_spider/core/scheduler.py

@@ -5,20 +5,22 @@ from idlelib.window import add_windows_to_menu
 from msilib import add_data
 
 from typing import Optional
-
+from baid_spider.utils.pqueue import SpiderPriorityQueue
 class Scheduler:
     def __init__(self):
-        self.requeste_queue :Optional[PriorityQueue] = None
+        self.requeste_queue :Optional[SpiderPriorityQueue] = None
 
     def open(self):
-        self.requeste_queue = PriorityQueue()
+        self.requeste_queue = SpiderPriorityQueue()
 
     async def next_request(self):
-        coro = self.requeste_queue.get()
-        try:
-            request = await asyncio.wait_for(coro,timeout=0.2)
-        except TimeoutError:
-            return None
+        # coro = self.requeste_queue.get()
+        # try:
+        #     request = await asyncio.wait_for(coro,timeout=0.2)
+        # except TimeoutError:
+        #     return None
+        # return request
+        request = await self.requeste_queue.get()
         return request
 
     async def enqueue_request(self,request):

baid_spider/utils/pqueue.py

@@ -0,0 +1,15 @@
+import asyncio
+
+from asyncio import PriorityQueue,TimeoutError
+
+class SpiderPriorityQueue(PriorityQueue):
+    def __init__(self,maxsize=0):
+        super(SpiderPriorityQueue,self).__init__(maxsize=maxsize)
+
+    async def get(self):
+        f = super().get()
+        try:
+            request = await asyncio.wait_for(f,timeout=0.2)
+        except TimeoutError:
+            return None
+        return request