优化请求未完成

baid_spider/__init__.py

@@ -0,0 +1 @@
+from baid_spider.http.request import Request

baid_spider/core/downloader.py

@@ -1,6 +1,8 @@
 import time
 
 import asyncio
+
+import aiohttp
 import requests
 
 
@@ -9,9 +11,13 @@ class Downloader:
     def __init__(self):
         pass
 
-    async def download(self,url):
-        # response = requests.get(url)
-        # print(response)
+    async def fetch(self,request):
+        await self.download(request)
 
-        await asyncio.sleep(0.1)
-        print('aaaa')
+
+    async def download(self,request):
+        response = requests.get(request.url)
+        return response
+
+        # await asyncio.sleep(0.1)
+        # print('aaaa')

baid_spider/core/engine.py

@@ -1,6 +1,10 @@
-from typing import Optional,Generator
+from typing import Optional,Generator,Callable
 from baid_spider.core.downloader import Downloader
 from baid_spider.core.scheduler import Scheduler
+import asyncio
+
+
+
 class Engine:
 
     def __init__(self):
@@ -14,9 +18,14 @@ class Engine:
             self.scheduler.open()
         self.downloader = Downloader()
         self.start_requests = iter(spider.start_requests())
-        await self.crawl()
+        await self._open_spider()
         #self.downloader.download(start_url)
 
+    async def _open_spider(self):
+        crowling = asyncio.create_task(self.crawl())
+        #这里可以做其它事情
+        await crowling
+
     async def crawl(self):
         '''主逻辑'''
         while True:
@@ -34,7 +43,13 @@ class Engine:
                     #入队
                     await self.enqueue_request(start_request)
     async def _crawl(self,request):
-        await self.downloader.download(request)
+        # todo 实现并发
+        await self._fetch(request)
+
+    async def _fetch(self,request):
+        await self.downloader.fetch(request)
+        callback: Callable = request.callback
+
 
     async def enqueue_request(self,request):
         # 入队

baid_spider/core/scheduler.py

@@ -2,7 +2,7 @@
 import asyncio
 from asyncio import PriorityQueue
 from idlelib.window import add_windows_to_menu
-from msilib import add_data
+
 
 from typing import Optional
 from baid_spider.utils.pqueue import SpiderPriorityQueue

baid_spider/http/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2026/6/10 20:28
+# @Author  : cmk
+# @File    : __init__.py.py
+# @Email   : 15726649712@163.com

baid_spider/http/request.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 2026/6/10 20:28
+# @Author  : cmk
+# @File    : request.py
+# @Email   : 15726649712@163.com
+
+
+from typing import Optional,Callable
+
+
+class Request:
+    def __init__(
+            self, url: str, *,
+            headers: Optional[dict] = None,
+            priority: Optional[int] = None,
+            timeout: Optional[int] = None,
+            method: Optional[str] = None,
+            cookies: Optional[dict] = None,
+            proxies: Optional[dict] = None,
+            proxy: Optional[dict] = None,
+            body: Optional[dict] = None,
+            json: Optional[dict] = None,
+            callback: Optional[Callable] = None,
+    ):
+        self.url = url
+        self.headers = headers
+        self.priority = priority
+        self.timeout = timeout
+        self.method = method
+        self.cookies = cookies
+        self.proxies = proxies
+        self.proxy = proxy
+        self.body = body
+        self.json = json
+        self.callback = callback

baid_spider/spider/__init__.py

@@ -1,3 +1,5 @@
+from baid_spider import Request
+
 
 class Spider:
 
@@ -8,7 +10,7 @@ class Spider:
     def start_requests(self):
         if self.start_urls:
             for url in self.start_urls:
-                yield url
+                yield Request(url=url)
         else:
             if hasattr(self,'start_url') and isinstance(getattr(self,'start_url'),str):
                 yield  getattr(self,'start_url')