Add URL to listing schema and normalise price for 'per week' listings

Author: TheJokersThief

daft_scraper/cli/__init__.py

@@ -18,7 +18,7 @@
 @app.command()
 def search(
     search_type: SearchType,
-    headers: List[str] = ['id', 'price', 'title', 'propertyType'],
+    headers: List[str] = ['id', 'price', 'title', 'propertyType', 'url'],
     location: List[str] = [Location.ALL.value],
     max_pages: int = sys.maxsize,
     min_price: int = 0,

daft_scraper/listing.py

@@ -1,5 +1,5 @@
 import re
-from marshmallow import Schema, fields, INCLUDE
+from marshmallow import Schema, fields, INCLUDE, post_load
 from marshmallow.utils import missing
 
 
@@ -52,6 +52,7 @@ class ListingPRS(Schema):
 
 
 class Listing(Schema):
+    URL_BASE = "https://daft.ie"
     PRICE_RE = re.compile(r'[0-9,]+')
 
     class Meta:
@@ -61,7 +62,10 @@ class Meta:
     def convert_price(self, value):
         matches = self.PRICE_RE.findall(value)
         if matches:
-            return int(matches[0].replace(',', ''))
+            price_int = int(matches[0].replace(',', ''))
+            if "week" in value:
+                price_int *= 4.34
+            return price_int
         return missing
 
     def convert_bed_and_bath(self, value):
@@ -70,6 +74,14 @@ def convert_bed_and_bath(self, value):
             return int(matches[0])
         return missing
 
+    def get_url(self, seo_friendly_path):
+        return "".join([self.URL_BASE, seo_friendly_path])
+
+    @post_load
+    def post_load(self, data, **kwargs):
+        data['url'] = self.get_url(data['seoFriendlyPath'])
+        return data
+
     id = fields.Int()
     title = fields.Str()
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "daft-scraper"
-version = "1.0.1"
+version = "1.0.2"
 description = "A webscraper for Daft.ie"
 authors = ["Evan Smith <[email protected]>"]
 license = "MIT"

tests/fixtures/listing.json

@@ -13,7 +13,7 @@
         ],
         "featuredLevel": "FEATURED",
         "publishDate": 1607001238000,
-        "price": "From \u20ac3,328 per month",
+        "price": "From \u20ac3,328 per week",
         "abbreviatedPrice": "\u20ac3,330+",
         "numBedrooms": "3 bed",
         "propertyType": "Apartments",

tests/test_listing.py

@@ -20,7 +20,7 @@ def test_confirm_next_json_parseable(self):
             got = Listing().load(json.load(fixture)['listing'])
 
         self.assertEqual(got.get('id'), 2315059)
-        self.assertEqual(got.get('price'), 3328)
+        self.assertEqual(got.get('price'), 14443.52)
         self.assertEqual(got.get('numBedrooms'), 3)
         self.assertEqual(got.get('numBathrooms'), None)