请求回调模块

baid_spider/core/downloader.py

@@ -1,8 +1,6 @@
 import time
 
 import asyncio
-
-import aiohttp
 import requests
 
 
@@ -12,11 +10,12 @@ class Downloader:
         pass
 
     async def fetch(self,request):
-        await self.download(request)
+        return await self.download(request)
 
 
     async def download(self,request):
         response = requests.get(request.url)
+        print(response)
         return response
 
         # await asyncio.sleep(0.1)

baid_spider/core/engine.py

@@ -2,17 +2,21 @@ from typing import Optional,Generator,Callable
 from baid_spider.core.downloader import Downloader
 from baid_spider.core.scheduler import Scheduler
 import asyncio
-
+from baid_spider.spider import Spider
+from inspect import iscoroutine
+from baid_spider.utils.spider import transform
 
 
 class Engine:
 
     def __init__(self):
-        self.downloader :Optional[Downloader] = Downloader()
+        self.downloader :Optional[Downloader] = None
         self.scheduler: Optional[Scheduler] = None
         self.start_requests: Optional[Generator] = None
+        self.spider: Optional[Spider] = None
 
     async def start_spider(self,spider):
+        self.spider = spider
         self.scheduler = Scheduler()
         if hasattr(self.scheduler,'open'):
             self.scheduler.open()
@@ -42,14 +46,31 @@ class Engine:
                 else:
                     #入队
                     await self.enqueue_request(start_request)
+
     async def _crawl(self,request):
         # todo 实现并发
-        await self._fetch(request)
+        outputs = await self._fetch(request)
+        # 处理outputs
+        if outputs:
+            async for output in outputs:
+                print(output)
+
+
 
     async def _fetch(self,request):
-        await self.downloader.fetch(request)
-        callback: Callable = request.callback
-
+        async def _seccess(_respones):
+            callback: Callable = request.callback or self.spider.parse
+            if _outputs := callback(_response):
+                if iscoroutine(_outputs):
+                    print(iscoroutine(_outputs))
+                    await  _outputs
+                else:
+                    return transform(_outputs)
+            return callback(_response)
+        _response = await self.downloader.fetch(request)
+        #callback: Callable = request.callback or self.spider.parse
+        outputs = await _seccess(_response)
+        return outputs
 
     async def enqueue_request(self,request):
         # 入队

baid_spider/http/request.py

@@ -6,21 +6,21 @@
 # @Email   : 15726649712@163.com
 
 
-from typing import Optional,Callable
+from typing import Optional,Callable,Dict
 
 
 class Request:
     def __init__(
             self, url: str, *,
-            headers: Optional[dict] = None,
-            priority: Optional[int] = None,
+            headers: Optional[Dict] = None,
+            priority: Optional[int] = 0,
             timeout: Optional[int] = None,
-            method: Optional[str] = None,
-            cookies: Optional[dict] = None,
-            proxies: Optional[dict] = None,
-            proxy: Optional[dict] = None,
-            body: Optional[dict] = None,
-            json: Optional[dict] = None,
+            method: Optional[str] = 'GET',
+            cookies: Optional[Dict] = None,
+            proxies: Optional[Dict] = None,
+            proxy: Optional[Dict] = None,
+            body: Optional[Dict] = None,
+            json: Optional[Dict] = None,
             callback: Optional[Callable] = None,
     ):
         self.url = url

baid_spider/spider/__init__.py

@@ -14,3 +14,6 @@ class Spider:
         else:
             if hasattr(self,'start_url') and isinstance(getattr(self,'start_url'),str):
                 yield  getattr(self,'start_url')
+
+    def parse(self,response):
+        raise NotImplementedError

baid_spider/utils/execptions.py

@@ -0,0 +1,3 @@
+
+class TransformTypeError(TypeError):
+    pass

baid_spider/utils/spider.py

@@ -0,0 +1,22 @@
+from inspect import isasyncgen,isgenerator
+
+from baid_spider.utils.execptions import TransformTypeError
+
+
+async def transform(func_result):
+    if isgenerator(func_result):
+        try:
+            for r in func_result:
+                yield r
+        except Exception:
+            print("同步生成器迭代过程发生异常")
+            raise
+    elif isasyncgen(func_result):
+        try:
+            async for r in func_result:
+                yield r
+        except Exception:
+            print("异步生成器迭代过程发生异常")
+            raise
+    else:
+        raise TransformTypeError('callback return value must be `generator` or `async generator`')

test/baidu_spider/baidu.py

@@ -1,8 +1,16 @@
+from baid_spider import Request
 from baid_spider.spider import Spider
 class Baidu_Spider(Spider):
 
     start_urls = ['https://www.baidu.com','https://www.baidu.com']
 
+    def parse(self,response):
+        print(response)
+        for i in range(10):
+            url = 'https://www.baidu.com'
+            response = Request(url=url)
+            yield response
+
     # def start_requests(self):
     #
     #     # response = requests.get(self.start_urls)