From 928bd4fb76863de55f7b45a316a441f2006dd256 Mon Sep 17 00:00:00 2001 From: Ulrich Schmid Date: Mon, 7 Nov 2016 11:41:25 +0100 Subject: [PATCH] Added request metaparameter. This allows for more fine-tuned applicability of deltafetch. Even if deltafetch wants to skip a request, setting this parameter will force it to go through. Example: scrapy.Request(url=myUrl, meta={'deltafetch_passthrough':True}) --- scrapy_deltafetch/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy_deltafetch/middleware.py b/scrapy_deltafetch/middleware.py index edcaae7..533fc09 100644 --- a/scrapy_deltafetch/middleware.py +++ b/scrapy_deltafetch/middleware.py @@ -74,7 +74,7 @@ def spider_closed(self, spider): def process_spider_output(self, response, result, spider): for r in result: - if isinstance(r, Request): + if isinstance(r, Request) and not r.meta.get('deltafetch_passthrough') == True: key = self._get_key(r) if self.db.has_key(key): logger.info("Ignoring already visited: %s" % r)