优化循环请求获取不到数据

2026-06-10 15:58:46 +08:00
parent 290ff82ab4
commit 29dd500118
1 changed files with 9 additions and 2 deletions
@@ -14,5 +14,12 @@ class Engine:

    def crawl(self):
        '''主逻辑'''
-        start_request = next(self.start_requests)
-        self.downloader.download(start_request)
+        while True:
+            try:
+                start_request = next(self.start_requests)
+                self.downloader.download(start_request)
+            except StopIteration:
+                self.start_requests = None
+            except Exception as exc:
+                #print(exc)
+                break