Docs: Updated Usage instructions and added module docs

Author: MeNsaaH

README.md

@@ -83,33 +83,40 @@ Found on [Read the Docs](https://search-engine-parser.readthedocs.io/en/latest)
 Query Results can be scraped from popular search engines as shown in the example snippet below
 
 ```python
-    from search_engine_parser.engines.yahoo import Search as YahooSearch
-    from search_engine_parser.engines.google import Search as GoogleSearch
-    from search_engine_parser.engines.bing import Search as BingSearch
-    import pprint
-
-    search_args = ('preaching to the choir', 1)
-    gsearch = GoogleSearch()
-    ysearch = YahooSearch()
-    bsearch = BingSearch()
-    gresults = gsearch.search(*search_args)
-    yresults = ysearch.search(*search_args)
-    bresults = bsearch.search(*search_args)
-    a = {
-        "Google": gresults,
-        "Yahoo": yresults,
-        "Bing": bresults}
-    # pretty print the result from each engine
-    for k, v in a.items():
-        print(f"-------------{k}------------")
-            pprint.pprint(v)
-
-    # print first title from google search
-    print(gresults["titles"][0])
-    # print 10th link from yahoo search
-    print(yresults["links"][9])
-    # print 6th description from bing search
-    print(bresults["descriptions"][5])
+  import pprint
+
+  from search_engine_parser.core.engines.bing import Search as BingSearch
+  from search_engine_parser.core.engines.google import Search as GoogleSearch
+  from search_engine_parser.core.engines.yahoo import Search as YahooSearch
+
+  search_args = ('preaching to the choir', 1)
+  gsearch = GoogleSearch()
+  ysearch = YahooSearch()
+  bsearch = BingSearch()
+  gresults = gsearch.search(*search_args)
+  yresults = ysearch.search(*search_args)
+  bresults = bsearch.search(*search_args)
+  a = {
+      "Google": gresults,
+      "Yahoo": yresults,
+      "Bing": bresults
+      }
+
+  # pretty print the result from each engine
+  for k, v in a.items():
+      print(f"-------------{k}------------")
+      for result in v:
+          pprint.pprint(result)
+
+  # print first title from google search
+  print(gresults["titles"][0])
+  # print 10th link from yahoo search
+  print(yresults["links"][9])
+  # print 6th description from bing search
+  print(bresults["descriptions"][5])
+
+  # print first result containing links, descriptions and title
+  print(gresults[0])
 ```
 
 For localization, you can pass the `url` keyword and a localized url. This would use the url to query and parse using the same engine's parser
@@ -118,6 +125,26 @@ For localization, you can pass the `url` keyword and a localized url. This would
   results = gsearch.search(*search_args, url="google.de")
 ```
 
+#### Async
+search-engine-parser supports `async` hence you could use codes like
+```python
+   results = await gsearch.async_search(*search_args)
+```
+
+#### Results
+The `SearchResults` after the searching
+```python
+  >>> results = gsearch.search("preaching the choir", 1)
+  >>> results
+  <search_engine_parser.core.base.SearchResult object at 0x7f907426a280>
+  # The object supports retreiving individual results by iteration of just by type (links, descriptions, titles)
+  >>> results[0] # Returns the first <SearchItem>
+  >>> results[0]["description"] # Get the description of the first item
+  >>> results[0]["link"] # get the link of the first item
+  >>> results["descriptions"] # Returns a list of all descriptions from all results
+```
+It can be iterated like a normal list to return individual SearchItem
+
 ### Command line
 
 Search engine parser comes with a CLI tool known as `pysearch` e.g

docs/engines.md

@@ -38,19 +38,26 @@ The engine modules are in the [search_engine_parser/core/engines/](https://githu
         # name of the engine to be displayed on the CLI, preferably PascalCase
         name = "FakeEngine"
         # engine url to be search, with parameters to be formatted e.g query , page
-        search_url = "https://search.fake.com/fake/search?q={query}&page={page}"
+        search_url = "https://search.fake.com/fake/search"
         # a short 2 or 3 line summary of the engine with some statistics, preferably obtained from wikipedia
         summary = "\t According to netmarketshare, this site is balderdash among "\
 	      "search engines with a market share that is close to 100%. "\
 	      "The fake engine includes many popular features but was solely created to show you an example ."
 
 
+        # this function should return the dict of params to be passed to the search_url
+        def get_params(self, query=None, page=None, offset=None, **kwargs):
+          params = {}
+          params["q"] =query
+          params["page"] = page
+          return params
+
         # This function should use beautiful soup (combined with regex if necessary) 
         # to return all the divs containiing results
         def parse_soup(self, soup):
             return soup.find_all('div', class_='fake-result-div')
 
-        # This function should parse each div to return title, link, and description 
+        # This function should parse each result soup to return title, link, and description 
         # NOTE: The implementation may not be as straightforward as shown below
         def parse_single_result(self, single_result):
             title_div = single_result.find('div', class_='fake-title')
@@ -69,32 +76,13 @@ The engine modules are in the [search_engine_parser/core/engines/](https://githu
 
 * Import the engine by adding to the following files
 
-[search_engine_parser/core/engines/__init__.py](https://github.com/bisoncorps/search-engine-parser/blob/master/search_engine_parser/core/engines/__init__.py)
+[search_engine_parser/__init__.py](https://github.com/bisoncorps/search-engine-parser/blob/master/search_engine_parser/__init__.py)
 
 ```python
     ...
-    from .fake import FakeEngineSearch
-```
-
-[search_engine_parser/core/__init__.py](https://github.com/bisoncorps/search-engine-parser/blob/master/search_engine_parser/core/__init__.py)
-
-```python
-    from search_engine_parser.core.engines import (
-        ...
-        FakeEngineSearch
-    )
+    from search_engine_parser.core.engines.fake import Search as FakeEngineSearch
 ```
 
-* Write Tests for the Engine to the [search_engine_parser/test/](https://github.com/bisoncorps/search-engine-parser/blob/master/search_engine_parser/test) directory
-
-* Include into the CLI at [search_engine_parser/core/cli.py](https://github.com/bisoncorps/search-engine-parser/blob/master/search_engine_parser/core/cli.py)
-
-```python
-    def main(args):
-        ...
-        elif engine == 'fake':
-            engine_class = FakeEngineSearch
-```
 
 * Make sure to write code documentation by following the [documentation guide](https://github.com/bisoncorps/search-engine-parser/blob/master/docs/documentation.md#documenting-an-engine)
 

search_engine_parser/__init__.py

@@ -21,6 +21,8 @@
 
 """
 
+# Allow import using `search_engine_parser.engines`
+from search_engine_parser.core import engines
 # Support for older versions of imports
 # DEPRECATION_WARNING: These imports will be removed in later versions
 from search_engine_parser.core.engines.aol import Search as AolSearch

search_engine_parser/core/base.py

@@ -26,10 +26,41 @@ class ReturnType(Enum):
 
 # All results returned are each items of search
 class SearchItem(dict):
-    pass
+    """ 
+    SearchItem is a dict of results containing keys (titles, descriptions, links and other 
+    additional keys dependending on the engine)
+    >>> result
+    <search_engine_parser.core.base.SearchItem object at 0x7f907426a280>
+    >>> result["description"]
+    Some description
+    >>> result["descriptions"]
+    Same description
+    """
+    def __getitem__(self, value):
+        """ Allow getting by index and by type ('descriptions', 'links'...)"""
+        try:
+            return super().__getitem__(value)
+        except KeyError:
+            pass
+        if not value.endswith('s'):
+            value += 's'
+        return super().__getitem__(value)
 
 
 class SearchResult():
+    """ 
+    The SearchResults after the searching
+
+    >>> results = gsearch.search("preaching the choir", 1)
+    >>> results
+    <search_engine_parser.core.base.SearchResult object at 0x7f907426a280>
+
+    The object supports retreiving individual results by iteration of just by type
+    >>> results[0] # Returns the first result <SearchItem>
+    >>> results["descriptions"] # Returns a list of all descriptions from all results
+
+    It can be iterated like a normal list to return individual SearchItem
+    """
     # Hold the results
     results = []
     # This method is inefficient, it will be in Deprecation soon
@@ -54,8 +85,11 @@ def keys(self):
             keys = x.keys()
         return keys 
 
-    def __len__(self):
-       return len(self.results)
+    def __len__(self): 
+        return len(self.results)
+
+    def __repr_(self):
+        return "<SearchResult: {} results>".format(len(self.results))
 
 
 class BaseSearch:
@@ -188,7 +222,7 @@ def get_results(self, soup, **kwargs):
 
         return search_results
 
-    def search(self, query=None, page=None, **kwargs):
+    def search(self, query=None, page=1, **kwargs):
         """
         Query the search engine
 
@@ -206,7 +240,7 @@ def search(self, query=None, page=None, **kwargs):
                     query, page, **kwargs)))
         return self.get_results(soup, **kwargs)
 
-    async def async_search(self, query=None, page=None, callback=None, **kwargs):
+    async def async_search(self, query=None, page=1, callback=None, **kwargs):
         """
         Query the search engine but in async mode