DAFT-10: Change listing to object and get ad page info (county, description, etc.) (TheJokersThief#10)

Author: TheJokersThief

daft_scraper/listing.py

@@ -1,7 +1,10 @@
+import json
 import re
 from marshmallow import Schema, fields, INCLUDE, post_load
 from marshmallow.utils import missing
 
+from daft_scraper import Daft
+
 
 class Seller(Schema):
     class Meta:
@@ -64,14 +67,14 @@ class Meta:
         unknown = INCLUDE
 
     totalUnitTypes = fields.Int()
-    subUnits = fields.List(fields.Nested(lambda: Listing()))
+    subUnits = fields.List(fields.Nested(lambda: ListingSchema()))
     tagLine = fields.Str()
     location = fields.Str()
     aboutDevelopment = fields.Str()
     brochure = fields.Str()
 
 
-class Listing(Schema):
+class ListingSchema(Schema):
     URL_BASE = "https://daft.ie"
     PRICE_RE = re.compile(r'[0-9,]+')
 
@@ -127,3 +130,34 @@ def post_load(self, data, **kwargs):
     image = fields.Dict(keys=fields.Str(), values=fields.Str())
     ber = fields.Nested(ListingBER, default=ListingBER())
     prs = fields.Nested(ListingPRS, default=ListingPRS())
+
+
+class Listing(dict):
+    _ad_page_info = None
+
+    def __init__(self, data: dict):
+        self.__dict__ = data
+
+    @property
+    def ad_page_info(self):
+        if not self._ad_page_info:
+            parsed_page = Daft().get(self.url)
+            script_text = parsed_page.find('script', {'id': '__NEXT_DATA__'})
+            self._ad_page_info = json.loads(script_text.string)
+        return self._ad_page_info
+
+    @property
+    def description(self) -> str:
+        return self.ad_page_info['props']['pageProps']['listing'].get('description', None)
+
+    @property
+    def county(self) -> list:
+        return self.ad_page_info['props']['pageProps']['dfpTargetingValues'].get('countyName', [])
+
+    @property
+    def area(self) -> list:
+        return self.ad_page_info['props']['pageProps']['dfpTargetingValues'].get('areaName', [])
+
+    @property
+    def views(self) -> int:
+        return self.ad_page_info['props']['pageProps'].get('listingViews', None)

daft_scraper/search/__init__.py

@@ -5,7 +5,7 @@
 from typing import List
 
 from daft_scraper import Daft
-from daft_scraper.listing import Listing
+from daft_scraper.listing import Listing, ListingSchema
 from daft_scraper.search.options import Option, PriceOption, SalePriceOption
 
 
@@ -39,7 +39,6 @@ def search(self, query: List[Option], max_pages: int = sys.maxsize):
             path = path.replace('ireland', locations[0])
             del options['location']
 
-        print(path, options)
         # Init pagination params
         options['pageSize'] = self.PAGE_SIZE
         options['from'] = 0
@@ -87,7 +86,7 @@ def _get_page_data(self, path, params):
     def _get_listings(self, listings: dict):
         """Convert a dict of listings into marshalled objects"""
         return [
-            Listing().load(listing['listing'])
+            Listing(ListingSchema().load(listing['listing']))
             for listing in listings
         ]
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "daft-scraper"
-version = "1.1.0"
+version = "1.2.0"
 description = "A webscraper for Daft.ie"
 authors = ["Evan Smith <[email protected]>"]
 license = "MIT"