Skip to content

Resolve request_fingerprint deprecation removal in Scrapy 2.12.0 #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 7 commits into from
Closed

Resolve request_fingerprint deprecation removal in Scrapy 2.12.0 #51

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3c68465
Resolve request_fingerprint deprecation
fsmeraldi Feb 24, 2025
b66c43d
Get fingerprint function through crawler, import legacy function othe…
fsmeraldi Feb 25, 2025
0bd04d8
Move the logic to from_crawler to minimize backward incompatibility
Gallaecio Feb 25, 2025
a547f65
Fix CI
Gallaecio Feb 25, 2025
a43a7af
Disable fail-fast in CI to fidn out which Python versions work
Gallaecio Feb 25, 2025
d656da2
CI: Update Python setup
Gallaecio Feb 25, 2025
f918c71
Drop support for end-of-life Python versions
Gallaecio Feb 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Resolve request_fingerprint deprecation
  • Loading branch information
@fsmeraldi
fsmeraldi committed Feb 24, 2025
commit 3c68465bf6b33cc470e57f58aedf47b9e42ec028
9 changes: 5 additions & 4 deletions scrapy_deltafetch/middleware.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from scrapy.http import Request
from scrapy.item import Item
from scrapy.utils.request import request_fingerprint
from scrapy.utils.request import RequestFingerprinter
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes
from scrapy.exceptions import NotConfigured
Expand All @@ -26,10 +26,11 @@ class DeltaFetch(object):
intensive).
"""

def __init__(self, dir, reset=False, stats=None):
def __init__(self, dir, reset=False, stats=None, crawler=None):
self.dir = dir
self.reset = reset
self.stats = stats
self.fingerprint=RequestFingerprinter(crawler).fingerprint

@classmethod
def from_crawler(cls, crawler):
Expand All @@ -38,7 +39,7 @@ def from_crawler(cls, crawler):
raise NotConfigured
dir = data_path(s.get('DELTAFETCH_DIR', 'deltafetch'))
reset = s.getbool('DELTAFETCH_RESET')
o = cls(dir, reset, crawler.stats)
o = cls(dir, reset, crawler.stats, crawler)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
Expand Down Expand Up @@ -79,7 +80,7 @@ def process_spider_output(self, response, result, spider):
yield r

def _get_key(self, request):
key = request.meta.get('deltafetch_key') or request_fingerprint(request)
key = request.meta.get('deltafetch_key') or self.fingerprint(request)
return to_bytes(key)

def _is_enabled_for_request(self, request):
Expand Down
6 changes: 4 additions & 2 deletions tests/test_deltafetch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from scrapy.spiders import Spider
from scrapy.settings import Settings
from scrapy.exceptions import NotConfigured
from scrapy.utils.request import request_fingerprint
from scrapy.utils.request import RequestFingerprinter
from scrapy.utils.python import to_bytes
from scrapy.statscollectors import StatsCollector
from scrapy.utils.test import get_crawler
Expand Down Expand Up @@ -318,8 +318,10 @@ def __init__(self, dir, reset=False, *args, **kwargs):
def test_get_key(self):
mw = self.mwcls(self.temp_dir, reset=True)
test_req1 = Request('http://url1')
crawler = get_crawler(Spider)
fingerprint=RequestFingerprinter(crawler).fingerprint
self.assertEqual(mw._get_key(test_req1),
to_bytes(request_fingerprint(test_req1)))
to_bytes(fingerprint(test_req1)))
test_req2 = Request('http://url2', meta={'deltafetch_key': b'dfkey1'})
self.assertEqual(mw._get_key(test_req2), b'dfkey1')

Expand Down