Added engine localization. Closes #88

Author: MeNsaaH

README.md

@@ -112,6 +112,12 @@ Query Results can be scraped from popular search engines as shown in the example
     print(bresults["descriptions"][5])
 ```
 
+For localization, you can pass the `url` keyword and a localized url. This would use the url to query and parse using the same engine's parser
+```python
+  # Use google.de instead of google.com
+  results = gsearch.search(*search_args, url="google.de")
+```
+
 ### Command line
 
 Search engine parser comes with a CLI tool known as `pysearch` e.g
@@ -132,6 +138,8 @@ There is a needed argument for the CLI i.e `-e Engine` followed by either of two
 
 ```bash
 
+usage: pysearch [-h] [-u URL] [-e ENGINE] {search,summary} ...
+
 SearchEngineParser
 
 positional arguments:
@@ -141,9 +149,11 @@ positional arguments:
 
 optional arguments:
   -h, --help            show this help message and exit
+  -u URL, --url URL     A custom link to use as base url for search e.g
+                        google.de
   -e ENGINE, --engine ENGINE
                         Engine to use for parsing the query e.g google, yahoo,
-                        bing, duckduckgo (default: google)
+                        bing,duckduckgo (default: google)
 ```
 
 `summary` just shows the summary of each search engine added with descriptions on the return

search_engine_parser/core/base.py

@@ -51,7 +51,10 @@ def keys(self):
         with suppress(IndexError):
             x = self.results[0]
             keys = x.keys()
-        return keys
+        return keys 
+
+    def __len__(self):
+       return len(self.results)
 
 
 class BaseSearch:
@@ -79,7 +82,7 @@ def parse_soup(self, soup):
         raise NotImplementedError("subclasses must define method <parse_soup>")
 
     @abstractmethod
-    def parse_single_result(self, single_result):
+    def parse_single_result(self, single_result, **kwargs):
         """
         Every div/span containing a result is passed here to retrieve
         `title`, `link` and `descr`
@@ -152,8 +155,14 @@ def get_search_url(self, query=None, page=None, **kwargs):
             offset = (page * 10) - 9
             params = self.get_params(
                 query=query, page=page, offset=offset, **kwargs)
-            url = self.search_url + urlencode(params)
-            self._parsed_url = urlparse(url)
+            url = urlparse(self.search_url)
+            # For localization purposes, custom urls can be parsed for the same engine
+            # such as google.de and google.com
+            if kwargs.get("url"):
+                new_url = urlparse(kwargs.pop("url"))
+                url._replace(netloc=new_url.netloc)
+            self._parsed_url = url._replace(query=urlencode(params))
+
         return self._parsed_url.geturl()
 
     def get_results(self, soup, **kwargs):

search_engine_parser/core/cli.py

@@ -19,6 +19,7 @@ def display(results, term, **args):
     def print_one(kwargs):
         """ Print one result to the console """
         # Header
+        print(kwargs)
         if kwargs.get("titles"):
             print("\t{}".format(term.magenta(kwargs.pop("titles"))))
         if kwargs.get("links"):
@@ -41,15 +42,10 @@ def print_one(kwargs):
         # TODO Some more optimization might be need
         len_results = 0
         for i in results:
-            len_results = len(results[i])
-            break
-        for i in range(len_results):
-            result = {k: results[k][i] for k in results}
-            print_one(result)
+            print_one(i)
     else:
         rank = args["rank"]
-        result = {k: results[k][rank] for k in results}
-        print_one(result)
+        print_one(results[rank])
 
 
 
@@ -76,7 +72,7 @@ def main(args):  # pylint: disable=too-many-branches
     engine = engine_class()
     try:
         # Display full details: Header, Link, Description
-        results = engine.search(args['query'], args['page'], return_type=ReturnType(args["type"]))
+        results = engine.search(args['query'], args['page'], return_type=ReturnType(args["type"]), url=args.get("url"))
         display(results, term, type=args.get('type'), rank=args.get('rank'))
     except NoResultsOrTrafficError as exc:
         print('\n', '{}'.format(term.red(str(exc))))
@@ -86,7 +82,7 @@ def runner():
     """
     runner that handles parsing logic
     """
-    parser = argparse.ArgumentParser(description='SearchEngineParser')
+    parser = argparse.ArgumentParser(description='SearchEngineParser', prog="pysearch")
     parser.add_argument(
         '-e', '--engine',
         help='Engine to use for parsing the query e.g google, yahoo, bing,'
@@ -97,6 +93,11 @@ def runner():
 
     parser_search = subparsers.add_parser('search', help='search help')
 
+    parser_search.add_argument(
+        '-u',
+        '--url',
+        help='A custom link to use as base url for search e.g google.de')
+
     parser_search.add_argument(
         '-q',
         '--query',

search_engine_parser/core/engines/aol.py

@@ -22,7 +22,7 @@ def parse_soup(self, soup):
         # find all divs
         return soup.find_all('div', class_='algo-sr')
 
-    def parse_single_result(self, single_result, return_type=ReturnType.FULL):
+    def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
         """
         Parses the source code to return
 

search_engine_parser/core/engines/ask.py

@@ -34,7 +34,7 @@ def parse_soup(self, soup):
         # find all class_='PartialSearchResults-item' => each result
         return soup.find_all('div', class_="PartialSearchResults-item")
 
-    def parse_single_result(self, single_result, return_type=ReturnType.FULL):
+    def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
         """
         Parses the source code to return
 

search_engine_parser/core/engines/baidu.py

@@ -46,7 +46,7 @@ def parse_soup(self, soup):
 
         return soup.find_all('div', {'id': re.compile(r"^\d{1,2}")})
 
-    def parse_single_result(self, single_result, return_type=ReturnType.FULL):
+    def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
         """
         Parses the source code to return
 

search_engine_parser/core/engines/bing.py

@@ -31,7 +31,7 @@ def parse_soup(self, soup):
         # find all li tags
         return soup.find_all('li', class_='b_algo')
 
-    def parse_single_result(self, single_result, return_type=ReturnType.FULL):
+    def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
         """
         Parses the source code to return
 

search_engine_parser/core/engines/duckduckgo.py

@@ -36,7 +36,7 @@ def parse_soup(self, soup):
         # find all div tags
         return soup.find_all('div', class_='result')
 
-    def parse_single_result(self, single_result, return_type=ReturnType.FULL):
+    def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
         """
         Parses the source code to return
 

search_engine_parser/core/engines/github.py

@@ -61,7 +61,7 @@ def parse_soup(self, soup):
         elif self.type == "Commits":
             return soup.find_all('div', class_='commits-list-item')
 
-    def parse_single_result(self, single_result, return_type=ReturnType.FULL):
+    def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
         """
         Parses the source code to return
 

search_engine_parser/core/engines/google.py

@@ -32,76 +32,7 @@ def parse_soup(self, soup):
         # find all class_='g' => each result
         return soup.find_all('div', class_='g')
 
-    def parse_result(self, results, **kwargs):
-        """
-        Runs every entry on the page through parse_single_result
-
-        :param results: Result of main search to extract individual results
-        :type results: list[`bs4.element.ResultSet`]
-        :returns: dictionary. Containing lists of titles, links, descriptions, direct results and other possible\
-            returns.
-        :rtype: dict
-        """
-        search_results = dict()
-        for each in results:
-            try:
-                rdict = self.parse_single_result(each, **kwargs)
-                # Create a list for all keys in rdict if not exist, else
-                for key in rdict.keys():
-                    if key not in search_results.keys():
-                        search_results[key] = list([rdict[key]])
-                    else:
-                        search_results[key].append(rdict[key])
-            except Exception: #pylint: disable=invalid-name, broad-except
-                pass
-
-        direct_answer = self.parse_direct_answer(results[0])
-        if direct_answer is not None:
-            search_results['direct_answer'] = direct_answer
-
-        return search_results
-
-    def parse_direct_answer(self, single_result, return_type=ReturnType.FULL):
-        # returns empty string when there is no direct answer
-        if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
-            direct_answer = ''
-            if not single_result.find('span', class_='st'):
-                # example query: President of US
-                if single_result.find('div', class_='Z0LcW'):
-                    direct_answer = single_result.find('div', class_='Z0LcW').find('a').text
-                
-                # example query: 5+5
-                elif single_result.find('span', class_='qv3Wpe'):
-                    direct_answer = single_result.find('span', class_='qv3Wpe').text            
-                
-                # example query: Weather in dallas
-                elif single_result.find('div', id='wob_wc'):
-                    weather_status = single_result.find('span', id='wob_dc').text
-                    temperature = single_result.find('span', id='wob_tm').text
-                    unit = single_result.find('div', class_='wob-unit').find('span', class_='wob_t').text
-                    direct_answer = weather_status + ', ' + temperature + unit  
-                
-                # example query: 100 euros in pounds
-                elif single_result.find('span', class_='DFlfde SwHCTb'):
-                    direct_answer = single_result.find('span', class_='DFlfde SwHCTb').text + ' ' +single_result.find('span', class_='MWvIVe').text
-
-                # example query: US time
-                elif single_result.find('div', class_="gsrt vk_bk dDoNo"):
-                    direct_answer = single_result.find('div', class_='gsrt vk_bk dDoNo').text
-
-                # Christmas
-                elif single_result.find('div', class_="zCubwf"):
-                    direct_answer = single_result.find('div', class_="zCubwf").text
-
-            
-            elif not single_result.find('span', class_='st').text:
-                # example queris: How long shoud a car service take?, fastest animal
-                if single_result.find('div', class_='Z0LcW'):
-                    direct_answer = single_result.find('div', class_='Z0LcW').text
-        
-        return direct_answer
-
-    def parse_single_result(self, single_result, return_type=ReturnType.FULL):
+    def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
         """
         Parses the source code to return