Initial migration from scrapylib codebase

Including Python 3 porting by @nyov
from scrapinghub/scrapylib#67

Author: redapple

.bumpversion.cfg

@@ -0,0 +1,9 @@
+[bumpversion]
+current_version = 0.1.0
+commit = True
+tag = True
+
+[bumpversion:file:setup.py]
+
+[bumpversion:file:scrapy_querycleaner/__init__.py]
+

.coveragerc

@@ -0,0 +1,3 @@
+[run]
+branch = true
+source = scrapy_querycleaner

.travis.yml

@@ -0,0 +1,16 @@
+language: python
+python: 3.5
+
+sudo: false
+
+env:
+  matrix:
+  - TOXENV=py27
+  - TOXENV=py35
+
+install: pip install -U tox codecov
+
+script: tox
+
+after_success:
+  - codecov

CHANGES.rst

@@ -0,0 +1,12 @@
+Changes
+=======
+
+
+x.x.x (yyyy-mm-dd)
+------------------
+
+Initial release.
+
+This version is functionally equivalent to scrapylib's v1.7.0
+``scrapylib.querycleaner.QueryCleanerMiddleware``.
+

README.md

@@ -0,0 +1,96 @@
+===================
+scrapy-querycleaner
+===================
+
+.. image:: https://travis-ci.org/scrapy-plugins/scrapy-querycleaner.svg?branch=master
+    :target: https://travis-ci.org/scrapy-plugins/scrapy-querycleaner
+
+.. image:: https://codecov.io/gh/scrapy-plugins/scrapy-querycleaner/branch/master/graph/badge.svg
+  :target: https://codecov.io/gh/scrapy-plugins/scrapy-querycleaner
+
+This is a Scrapy spider middleware to clean up the request URL GET query parameters
+at the output of the spider in accordance with the patterns provided by the user.
+
+
+Installation
+============
+
+Install scrapy-querycleaner using ``pip``::
+
+    $ pip install scrapy-querycleaner
+
+
+Configuration
+=============
+
+1. Add ``QueryCleanerMiddleware`` by including it in ``SPIDER_MIDDLEWARES``
+   in your ``settings.py`` file::
+
+      SPIDER_MIDDLEWARES = {
+          'scrapy_querycleaner.QueryCleanerMiddleware': 100,
+      }
+
+   Here, priority ``100`` is just an example.
+   Set its value depending on other middlewares you may have enabled already.
+
+2. Enable the middleware using either ``QUERYCLEANER_REMOVE``
+   or ``QUERYCLEANER_KEEP`` (or both) in your ``setting.py``.
+
+
+Usage
+=====
+
+At least one of the following settings needs to be present for the
+middleware to be enabled.
+
+
+.. note::
+    You can specify a list of parameter names by using the ``|`` (*OR*) regex operator.
+
+    For example, the pattern ``search|login|postid`` will match query parameters *search*,
+    *login* and *postid*.
+    This is by far the most common usage case.
+
+    And by setting ``QUERYCLEANER_REMOVE`` value to ``.*``
+    you can completely remove all URL query parameters.
+
+
+Supported settings
+------------------
+
+``QUERYCLEANER_REMOVE``
+    a pattern (regular expression) that a query parameter name must match
+    in order to be removed from the URL. (All the others will be accepted.)
+
+``QUERYCLEANER_KEEP``
+    a pattern that a query parameter name must match in order to be kept in the URL.
+    (All the others will be removed.)
+
+You can combine both if some query parameters patterns should be kept and some should not.
+
+The **remove** pattern has precedence over the *keep* one.
+
+
+Example
+-------
+
+Let's suppose that the spider extracts URLs like::
+
+    http://www.example.com/product.php?pid=135&cid=12&ttda=12
+
+and we want to leave only the parameter ``pid``.
+
+To achieve this objective we can use either ``QUERYCLEANER_REMOVE``
+or ``QUERYCLEANER_KEEP``:
+
+- In the first case, the pattern would be ``cid|ttda``::
+
+    QUERYCLEANER_REMOVE = 'cid|ttda'
+
+- In the second case, ``pid``::
+
+    QUERYCLEANER_KEEP = 'pid'
+
+
+The best solution depends on a particular case, that is,
+how the query filters will affect any other URL that the spider is expected to extract.

README.rst

@@ -0,0 +1,2 @@
+scrapy>=1.0
+six

requirements.txt

@@ -0,0 +1,4 @@
+from .middleware import QueryCleanerMiddleware
+
+
+__version__ = "0.1.0"

scrapy_querycleaner/__init__.py

@@ -0,0 +1,81 @@
+"""Get parameter cleaner for AS.
+
+Add removed/kept pattern (regex) with
+
+QUERYCLEANER_REMOVE
+QUERYCLEANER_KEEP
+
+Remove patterns has precedence.
+"""
+import re
+from six.moves.urllib.parse import quote
+from six import string_types
+
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.http import Request
+from scrapy.exceptions import NotConfigured
+
+from w3lib.url import _safe_chars
+
+def _parse_query_string(query):
+    """Used for replacing cgi.parse_qsl.
+    The cgi version returns the same pair for query 'key'
+    and query 'key=', so reconstruction
+    maps to the same string. But some sites does not handle both versions
+    in the same way.
+    This version returns (key, None) in the first case, and (key, '') in the
+    second one, so correct reconstruction can be performed."""
+
+    params = query.split("&")
+    keyvals = []
+    for param in params:
+        kv = param.split("=") + [None]
+        keyvals.append((kv[0], kv[1]))
+    return keyvals
+
+def _filter_query(query, remove_re=None, keep_re=None):
+    """
+    Filters query parameters in a query string according to key patterns
+    >>> _filter_query('as=3&bs=8&cs=9')
+    'as=3&bs=8&cs=9'
+    >>> _filter_query('as=3&bs=8&cs=9', None, re.compile("as|bs"))
+    'as=3&bs=8'
+    >>> _filter_query('as=3&bs=8&cs=9', re.compile("as|bs"))
+    'cs=9'
+    >>> _filter_query('as=3&bs=8&cs=9', re.compile("as|bs"), re.compile("as|cs"))
+    'cs=9'
+    """
+    keyvals = _parse_query_string(query)
+    qargs = []
+    for k, v in keyvals:
+        if remove_re is not None and remove_re.search(k):
+            continue
+        if keep_re is None or keep_re.search(k):
+            qarg = quote(k, _safe_chars)
+            if isinstance(v, string_types):
+                qarg = qarg + '=' + quote(v, _safe_chars)
+            qargs.append(qarg.replace("%20", "+"))
+    return '&'.join(qargs)
+
+class QueryCleanerMiddleware(object):
+    def __init__(self, settings):
+        remove = settings.get("QUERYCLEANER_REMOVE")
+        keep = settings.get("QUERYCLEANER_KEEP")
+        if not (remove or keep):
+            raise NotConfigured
+        self.remove = re.compile(remove) if remove else None
+        self.keep = re.compile(keep) if keep else None
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler.settings)
+
+    def process_spider_output(self, response, result, spider):
+        for res in result:
+            if isinstance(res, Request):
+                parsed = urlparse_cached(res)
+                if parsed.query:
+                    parsed = parsed._replace(query=_filter_query(parsed.query, self.remove, self.keep))
+                    res = res.replace(url=parsed.geturl())
+            yield res
+

scrapy_querycleaner/middleware.py

@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1