DAFT-15: Allow an initial offset (TheJokersThief#17)

Author: TheJokersThief

daft_scraper/listing.py

@@ -2,6 +2,7 @@
 import re
 from marshmallow import Schema, fields, INCLUDE, post_load
 from marshmallow.utils import missing
+from urllib.parse import urljoin
 
 from daft_scraper import Daft
 
@@ -98,7 +99,7 @@ def convert_bed_and_bath(self, value):
         return missing
 
     def get_url(self, seo_friendly_path):
-        return "".join([self.URL_BASE, seo_friendly_path])
+        return urljoin(self.URL_BASE, seo_friendly_path)
 
     @post_load
     def post_load(self, data, **kwargs):

daft_scraper/search/__init__.py

@@ -2,13 +2,17 @@
 import sys
 from bs4 import BeautifulSoup
 from enum import Enum
-from typing import List
+from typing import List, Callable
 
 from daft_scraper import Daft
 from daft_scraper.listing import Listing, ListingSchema
 from daft_scraper.search.options import Option, PriceOption, SalePriceOption
 
 
+def empty_post_process_hook(listing: Listing, raw_data: dict) -> Listing:
+    return listing
+
+
 class SearchType(Enum):
     RENT = "property-for-rent"
     SALE = "property-for-sale"
@@ -22,16 +26,16 @@ class DaftSearch():
     PAGE_SIZE = 20
     SALE_TYPES = [SearchType.SALE, SearchType.NEW_HOMES, SearchType.COMMERCIAL_SALE]
 
-    def __init__(self, search_type: SearchType):
+    def __init__(self, search_type: SearchType, post_process_hook: Callable = empty_post_process_hook):
         self.search_type = search_type
+        self.post_process_hook = post_process_hook
         self.site = Daft()
 
-    def search(self, query: List[Option], max_pages: int = sys.maxsize):
+    def search(self, query: List[Option], max_pages: int = sys.maxsize, page_offset: int = 0):
         path = self._build_search_path()
 
         # Convert options to their string form
         options = self._translate_options(query)
-        listings = []
 
         # If only one location is specified, it should be in the URL, not the params
         locations = options.get('location', [])
@@ -41,7 +45,7 @@ def search(self, query: List[Option], max_pages: int = sys.maxsize):
 
         # Init pagination params
         options['pageSize'] = self.PAGE_SIZE
-        options['from'] = 0
+        options['from'] = self._calc_offset(page_offset)
 
         # Fetch the first page and get pagination info
         page_data = self._get_page_data(path, options)
@@ -50,14 +54,12 @@ def search(self, query: List[Option], max_pages: int = sys.maxsize):
 
         while current_page < min(totalPages, max_pages):
             listing_data = page_data['props']['pageProps']['listings']
-            listings.extend(self._get_listings(listing_data))
+            yield from self._get_listings(listing_data)
 
             options['from'] = self._calc_offset(current_page)
             page_data = self._get_page_data(path, options)
             current_page = page_data['props']['pageProps']['paging']['currentPage']
 
-        return listings
-
     def _build_search_path(self):
         """Build the URL path for searches"""
         return "/".join([self.search_type.value, "ireland"])
@@ -85,10 +87,11 @@ def _get_page_data(self, path, params):
 
     def _get_listings(self, listings: dict):
         """Convert a dict of listings into marshalled objects"""
-        return [
-            Listing(ListingSchema().load(listing['listing']))
-            for listing in listings
-        ]
+        for listing in listings:
+            yield self.post_process_hook(
+                Listing(ListingSchema().load(listing['listing'])),
+                listing
+            )
 
     def _calc_offset(self, current_page: int):
         """Calculate the offset for pagination"""

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "daft-scraper"
-version = "1.2.7"
+version = "1.3.0"
 description = "A webscraper for Daft.ie"
 authors = ["Evan Smith <[email protected]>"]
 license = "MIT"

tests/test_search.py

@@ -31,7 +31,7 @@ def test_search(self, *args):
             BedOption(1, 4),
         ]
 
-        got = self.api.search(options)
+        got = list(self.api.search(options))
         self.assertEqual(got[0].id, 1443907)
 
     def test__translate_options(self):
@@ -83,7 +83,7 @@ def test__get_listings(self, *args):
         path = self.api._build_search_path()
         page_data = self.api._get_page_data(path, params={})
 
-        got = self.api._get_listings(page_data['props']['pageProps']['listings'])
+        got = list(self.api._get_listings(page_data['props']['pageProps']['listings']))
         self.assertEqual(got[0].id, 1443907)
 
     def test__calc_offset(self):