From 9564b048a9720de8fbfd246977602dc56feef1ab Mon Sep 17 00:00:00 2001 From: cmk_B <15726649712@163.com> Date: Wed, 17 Jun 2026 16:53:25 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=B7=E6=B1=82=E5=9B=9E=E8=B0=83=E6=A8=A1?= =?UTF-8?q?=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- baid_spider/core/downloader.py | 5 ++--- baid_spider/core/engine.py | 33 +++++++++++++++++++++++++++------ baid_spider/http/request.py | 18 +++++++++--------- baid_spider/spider/__init__.py | 5 ++++- baid_spider/utils/execptions.py | 3 +++ baid_spider/utils/spider.py | 22 ++++++++++++++++++++++ test/baidu_spider/baidu.py | 8 ++++++++ 7 files changed, 75 insertions(+), 19 deletions(-) create mode 100644 baid_spider/utils/execptions.py create mode 100644 baid_spider/utils/spider.py diff --git a/baid_spider/core/downloader.py b/baid_spider/core/downloader.py index 9b8e93c..15ac753 100644 --- a/baid_spider/core/downloader.py +++ b/baid_spider/core/downloader.py @@ -1,8 +1,6 @@ import time import asyncio - -import aiohttp import requests @@ -12,11 +10,12 @@ class Downloader: pass async def fetch(self,request): - await self.download(request) + return await self.download(request) async def download(self,request): response = requests.get(request.url) + print(response) return response # await asyncio.sleep(0.1) diff --git a/baid_spider/core/engine.py b/baid_spider/core/engine.py index 4dac4fd..38e281f 100644 --- a/baid_spider/core/engine.py +++ b/baid_spider/core/engine.py @@ -2,17 +2,21 @@ from typing import Optional,Generator,Callable from baid_spider.core.downloader import Downloader from baid_spider.core.scheduler import Scheduler import asyncio - +from baid_spider.spider import Spider +from inspect import iscoroutine +from baid_spider.utils.spider import transform class Engine: def __init__(self): - self.downloader :Optional[Downloader] = Downloader() + self.downloader :Optional[Downloader] = None self.scheduler: Optional[Scheduler] = None self.start_requests: Optional[Generator] = None + self.spider: Optional[Spider] = None async def start_spider(self,spider): + self.spider = spider self.scheduler = Scheduler() if hasattr(self.scheduler,'open'): self.scheduler.open() @@ -42,14 +46,31 @@ class Engine: else: #入队 await self.enqueue_request(start_request) + async def _crawl(self,request): # todo 实现并发 - await self._fetch(request) + outputs = await self._fetch(request) + # 处理outputs + if outputs: + async for output in outputs: + print(output) + + async def _fetch(self,request): - await self.downloader.fetch(request) - callback: Callable = request.callback - + async def _seccess(_respones): + callback: Callable = request.callback or self.spider.parse + if _outputs := callback(_response): + if iscoroutine(_outputs): + print(iscoroutine(_outputs)) + await _outputs + else: + return transform(_outputs) + return callback(_response) + _response = await self.downloader.fetch(request) + #callback: Callable = request.callback or self.spider.parse + outputs = await _seccess(_response) + return outputs async def enqueue_request(self,request): # 入队 diff --git a/baid_spider/http/request.py b/baid_spider/http/request.py index b697b09..b57bf60 100644 --- a/baid_spider/http/request.py +++ b/baid_spider/http/request.py @@ -6,21 +6,21 @@ # @Email : 15726649712@163.com -from typing import Optional,Callable +from typing import Optional,Callable,Dict class Request: def __init__( self, url: str, *, - headers: Optional[dict] = None, - priority: Optional[int] = None, + headers: Optional[Dict] = None, + priority: Optional[int] = 0, timeout: Optional[int] = None, - method: Optional[str] = None, - cookies: Optional[dict] = None, - proxies: Optional[dict] = None, - proxy: Optional[dict] = None, - body: Optional[dict] = None, - json: Optional[dict] = None, + method: Optional[str] = 'GET', + cookies: Optional[Dict] = None, + proxies: Optional[Dict] = None, + proxy: Optional[Dict] = None, + body: Optional[Dict] = None, + json: Optional[Dict] = None, callback: Optional[Callable] = None, ): self.url = url diff --git a/baid_spider/spider/__init__.py b/baid_spider/spider/__init__.py index a864976..3d4db87 100644 --- a/baid_spider/spider/__init__.py +++ b/baid_spider/spider/__init__.py @@ -13,4 +13,7 @@ class Spider: yield Request(url=url) else: if hasattr(self,'start_url') and isinstance(getattr(self,'start_url'),str): - yield getattr(self,'start_url') \ No newline at end of file + yield getattr(self,'start_url') + + def parse(self,response): + raise NotImplementedError \ No newline at end of file diff --git a/baid_spider/utils/execptions.py b/baid_spider/utils/execptions.py new file mode 100644 index 0000000..71cb01a --- /dev/null +++ b/baid_spider/utils/execptions.py @@ -0,0 +1,3 @@ + +class TransformTypeError(TypeError): + pass \ No newline at end of file diff --git a/baid_spider/utils/spider.py b/baid_spider/utils/spider.py new file mode 100644 index 0000000..46c7318 --- /dev/null +++ b/baid_spider/utils/spider.py @@ -0,0 +1,22 @@ +from inspect import isasyncgen,isgenerator + +from baid_spider.utils.execptions import TransformTypeError + + +async def transform(func_result): + if isgenerator(func_result): + try: + for r in func_result: + yield r + except Exception: + print("同步生成器迭代过程发生异常") + raise + elif isasyncgen(func_result): + try: + async for r in func_result: + yield r + except Exception: + print("异步生成器迭代过程发生异常") + raise + else: + raise TransformTypeError('callback return value must be `generator` or `async generator`') \ No newline at end of file diff --git a/test/baidu_spider/baidu.py b/test/baidu_spider/baidu.py index 9f149d9..8f0c75e 100644 --- a/test/baidu_spider/baidu.py +++ b/test/baidu_spider/baidu.py @@ -1,8 +1,16 @@ +from baid_spider import Request from baid_spider.spider import Spider class Baidu_Spider(Spider): start_urls = ['https://www.baidu.com','https://www.baidu.com'] + def parse(self,response): + print(response) + for i in range(10): + url = 'https://www.baidu.com' + response = Request(url=url) + yield response + # def start_requests(self): # # # response = requests.get(self.start_urls)