85 lines
2.7 KiB
Python
85 lines
2.7 KiB
Python
from typing import Optional,Generator,Callable
|
|
from baid_spider.core.downloader import Downloader
|
|
from baid_spider.core.scheduler import Scheduler
|
|
import asyncio
|
|
from baid_spider.spider import Spider
|
|
from inspect import iscoroutine
|
|
from baid_spider.utils.spider import transform
|
|
|
|
|
|
class Engine:
|
|
|
|
def __init__(self):
|
|
self.downloader :Optional[Downloader] = None
|
|
self.scheduler: Optional[Scheduler] = None
|
|
self.start_requests: Optional[Generator] = None
|
|
self.spider: Optional[Spider] = None
|
|
|
|
async def start_spider(self,spider):
|
|
self.spider = spider
|
|
self.scheduler = Scheduler()
|
|
if hasattr(self.scheduler,'open'):
|
|
self.scheduler.open()
|
|
self.downloader = Downloader()
|
|
self.start_requests = iter(spider.start_requests())
|
|
await self._open_spider()
|
|
#self.downloader.download(start_url)
|
|
|
|
async def _open_spider(self):
|
|
crowling = asyncio.create_task(self.crawl())
|
|
#这里可以做其它事情
|
|
await crowling
|
|
|
|
async def crawl(self):
|
|
'''主逻辑'''
|
|
while True:
|
|
if (request :=await self._get_next_request()) is not None:
|
|
await self._crawl(request)
|
|
else:
|
|
try:
|
|
start_request = next(self.start_requests)
|
|
except StopIteration:
|
|
self.start_requests = None
|
|
except Exception as exc:
|
|
#print(exc)
|
|
break
|
|
else:
|
|
#入队
|
|
await self.enqueue_request(start_request)
|
|
|
|
async def _crawl(self,request):
|
|
# todo 实现并发
|
|
outputs = await self._fetch(request)
|
|
# 处理outputs
|
|
if outputs:
|
|
async for output in outputs:
|
|
print(output)
|
|
|
|
|
|
|
|
async def _fetch(self,request):
|
|
async def _seccess(_respones):
|
|
callback: Callable = request.callback or self.spider.parse
|
|
if _outputs := callback(_response):
|
|
if iscoroutine(_outputs):
|
|
print(iscoroutine(_outputs))
|
|
await _outputs
|
|
else:
|
|
return transform(_outputs)
|
|
return callback(_response)
|
|
_response = await self.downloader.fetch(request)
|
|
#callback: Callable = request.callback or self.spider.parse
|
|
outputs = await _seccess(_response)
|
|
return outputs
|
|
|
|
async def enqueue_request(self,request):
|
|
# 入队
|
|
await self._schedule_request(request)
|
|
|
|
|
|
async def _schedule_request(self,request):
|
|
#todo 去重
|
|
await self.scheduler.enqueue_request(request)
|
|
|
|
async def _get_next_request(self):
|
|
return await self.scheduler.next_request() |