初步队列完成

baid_spider/core/downloader.py

@@ -1,5 +1,6 @@
 import time
 
+import asyncio
 import requests
 
 
@@ -8,9 +9,9 @@ class Downloader:
     def __init__(self):
         pass
 
-    def download(self,url):
+    async def download(self,url):
         # response = requests.get(url)
         # print(response)
 
-        time.sleep(0.1)
+        await asyncio.sleep(0.1)
         print('aaaa')

baid_spider/core/engine.py

@@ -1,25 +1,49 @@
 from typing import Optional,Generator
 from baid_spider.core.downloader import Downloader
-
+from baid_spider.core.scheduler import Scheduler
 class Engine:
 
     def __init__(self):
         self.downloader :Optional[Downloader] = Downloader()
+        self.scheduler: Optional[Scheduler] = None
         self.start_requests: Optional[Generator] = None
 
-    def start_spider(self,spider):
+    async def start_spider(self,spider):
+        self.scheduler = Scheduler()
+        if hasattr(self.scheduler,'open'):
+            self.scheduler.open()
+        self.downloader = Downloader()
         self.start_requests = iter(spider.start_requests())
-        self.crawl()
+        await self.crawl()
         #self.downloader.download(start_url)
 
-    def crawl(self):
+    async def crawl(self):
         '''主逻辑'''
         while True:
-            try:
-                start_request = next(self.start_requests)
-                self.downloader.download(start_request)
-            except StopIteration:
-                self.start_requests = None
-            except Exception as exc:
-                #print(exc)
-                break
+            if (request :=await self._get_next_request()) is not None:
+                await self._crawl(request)
+            else:
+                try:
+                    start_request = next(self.start_requests)
+                except StopIteration:
+                    self.start_requests = None
+                except Exception as exc:
+                    #print(exc)
+                    break
+                else:
+                    #入队
+                    await self.enqueue_request(start_request)
+    async def _crawl(self,request):
+        await self.downloader.download(request)
+
+    async def enqueue_request(self,request):
+        # 入队
+        await self._schedule_request(request)
+
+
+    async def _schedule_request(self,request):
+        #todo 去重
+        await  self.scheduler.enqueue_request(request)
+
+    async def _get_next_request(self):
+        return await self.scheduler.next_request()

baid_spider/core/scheduler.py

@@ -0,0 +1,27 @@
+#队列
+import asyncio
+from asyncio import PriorityQueue
+from idlelib.window import add_windows_to_menu
+from msilib import add_data
+
+from typing import Optional
+
+class Scheduler:
+    def __init__(self):
+        self.requeste_queue :Optional[PriorityQueue] = None
+
+    def open(self):
+        self.requeste_queue = PriorityQueue()
+
+    async def next_request(self):
+        coro = self.requeste_queue.get()
+        try:
+            request = await asyncio.wait_for(coro,timeout=0.2)
+        except TimeoutError:
+            return None
+        return request
+
+    async def enqueue_request(self,request):
+        await self.requeste_queue.put(request)
+
+

test/baidu_spider/run.py

@@ -1,6 +1,11 @@
 from baid_spider.core.engine import Engine
 from baidu import Baidu_Spider
+import asyncio
 
-baiduspider = Baidu_Spider()
-engine = Engine()
-engine.start_spider(baiduspider)
+
+async def run():
+    baiduspider = Baidu_Spider()
+    engine = Engine()
+    await engine.start_spider(baiduspider)
+
+asyncio.run(run())