Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
replace BaseItem to Item (BaseItem deprecated in Scrapy 2.3)
  • Loading branch information
pawelmhm committed Jun 11, 2021
commit 95fcc1567528d0f524ba2a3154d2ebae0d878369
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -87,3 +87,4 @@ ENV/

# Rope project settings
.ropeproject
.idea
4 changes: 2 additions & 2 deletions scrapy_deltafetch/middleware.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import time

from scrapy.http import Request
from scrapy.item import BaseItem
from scrapy.item import Item
from scrapy.utils.request import request_fingerprint
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes
Expand Down Expand Up @@ -81,7 +81,7 @@ def process_spider_output(self, response, result, spider):
if self.stats:
self.stats.inc_value('deltafetch/skipped', spider=spider)
continue
elif isinstance(r, (BaseItem, dict)):
elif isinstance(r, (Item, dict)):
key = self._get_key(response.request)
self.db[key] = str(time.time())
if self.stats:
Expand Down
8 changes: 4 additions & 4 deletions tests/test_deltafetch.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import mock
import tempfile
from scrapy import Request
from scrapy.item import BaseItem
from scrapy.item import Item
from scrapy.spiders import Spider
from scrapy.settings import Settings
from scrapy.exceptions import NotConfigured
Expand Down Expand Up @@ -192,7 +192,7 @@ def test_process_spider_output(self):
b'test_key_2']))

# if the spider returns items, the request's key is added in db
result = [BaseItem(), "not a base item"]
result = [Item(), "not a base item"]
self.assertEqual(list(mw.process_spider_output(
response, result, self.spider)), result)
self.assertEqual(set(mw.db.keys()),
Expand Down Expand Up @@ -235,7 +235,7 @@ def test_process_spider_output_stats(self):
self.assertEqual(list(mw.process_spider_output(
response, result, self.spider)), [result[0]])
self.assertEqual(self.stats.get_value('deltafetch/skipped'), 1)
result = [BaseItem(), "not a base item"]
result = [Item(), "not a base item"]
self.assertEqual(list(mw.process_spider_output(
response, result, self.spider)), result)
self.assertEqual(self.stats.get_value('deltafetch/stored'), 1)
Expand Down Expand Up @@ -305,7 +305,7 @@ def __init__(self, dir, reset=False, *args, **kwargs):
response, result, self.spider)), [result[0]])
self.assertEqual(self.stats.get_value('deltafetch/skipped'), None)

result = [BaseItem(), "not a base item"]
result = [Item(), "not a base item"]
self.assertEqual(list(mw.process_spider_output(
response, result, self.spider)), result)
self.assertEqual(self.stats.get_value('deltafetch/stored'), None)
Expand Down