DAFT-9: Add retries to requests (TheJokersThief#9)

Author: TheJokersThief

daft_scraper/__init__.py

@@ -1,6 +1,7 @@
 import requests
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin
+from urllib3.util import Retry
 
 
 class DaftHTTPSession(requests.Session):
@@ -29,9 +30,17 @@ class Daft():
     """
     BASE_URL = "https://www.daft.ie/"
 
-    def __init__(self, user_agent=None):
+    def __init__(self, user_agent=None, enable_retries=True):
         self.site = DaftHTTPSession(Daft.BASE_URL, user_agent=user_agent)
 
+        if enable_retries:
+            retry_strategy = Retry(
+                total=3,
+                backoff_factor=0.1
+            )
+            adapter = requests.adapters.HTTPAdapter(max_retries=retry_strategy)
+            self.site.mount("https://", adapter)
+
     def get(self, url, params=None):
         req = self.site.get(url, params=params)
         req.raise_for_status()

daft_scraper/search/__init__.py

@@ -39,6 +39,7 @@ def search(self, query: List[Option], max_pages: int = sys.maxsize):
             path = path.replace('ireland', locations[0])
             del options['location']
 
+        print(path, options)
         # Init pagination params
         options['pageSize'] = self.PAGE_SIZE
         options['from'] = 0

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "daft-scraper"
-version = "1.0.4"
+version = "1.1.0"
 description = "A webscraper for Daft.ie"
 authors = ["Evan Smith <[email protected]>"]
 license = "MIT"
@@ -16,6 +16,7 @@ requests = "^2.25.0"
 typer = "^0.3.2"
 marshmallow = "^3.9.1"
 tabulate = "^0.8.7"
+urllib3 = "^1.26.2"
 
 [tool.poetry.dev-dependencies]
 flake8 = "^3.8.4"