Some PY3 ports

nyov · nyov · commit a04c42f37c85 · 2015-08-19T15:01:04.000Z
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,4 @@
+six>=1.5.2
 boto
 hubstorage
 python-dateutil
diff --git a/scrapylib/constraints/pipeline.py b/scrapylib/constraints/pipeline.py
@@ -6,6 +6,6 @@ def process_item(self, item, spider):
         try:
             for c in item.constraints:
                 c(item)
-        except AssertionError, e:
+        except AssertionError as e:
             raise DropItem(str(e))
         return item
diff --git a/scrapylib/magicfields.py b/scrapylib/magicfields.py
@@ -78,7 +78,7 @@ def _extract_regex_group(regex, txt):
         try:
             compiled = re.compile(regex)
             _REGEXES[regex] = compiled
-        except Exception, e:
+        except Exception as e:
             errmessage = e.message
             _REGEX_ERRORS[regex] = errmessage
     if errmessage:
@@ -143,7 +143,7 @@ def _format(fmt, spider, response, item, fixed_values):
         if regex:
             try:
                 out = _extract_regex_group(regex, out)
-            except ValueError, e:
+            except ValueError as e:
                 spider.log("Error at '%s': %s" % (m.group(), e.message))
 
     return out
diff --git a/scrapylib/processors/__init__.py b/scrapylib/processors/__init__.py
@@ -2,7 +2,7 @@
 import locale as localelib
 import re
 import time
-from urlparse import urljoin
+from six.moves.urllib.parse import urljoin
 
 from scrapy.loader.processors import MapCompose, TakeFirst
 from scrapy.utils.markup import (remove_tags, replace_escape_chars,
diff --git a/scrapylib/proxy.py b/scrapylib/proxy.py
@@ -1,7 +1,10 @@
 import base64
-from urllib import unquote
-from urllib2 import _parse_proxy
-from urlparse import urlunparse
+from six.moves.urllib.parse import unquote
+try:
+    from urllib2 import _parse_proxy
+except ImportError:
+    from urllib.request import _parse_proxy
+from six.moves.urllib.parse import urlunparse
 
 
 class SelectiveProxyMiddleware(object):
diff --git a/scrapylib/querycleaner.py b/scrapylib/querycleaner.py
@@ -8,7 +8,7 @@
 Remove patterns has precedence.
 """
 import re
-from urllib import quote
+from six.moves.urllib.parse import quote
 
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.http import Request
diff --git a/tests/test_magicfields.py b/tests/test_magicfields.py
@@ -1,3 +1,4 @@
+from __future__ import print_function
 import re, os
 from unittest import TestCase
 
@@ -26,7 +27,7 @@ def setUp(self):
         self.spider = Spider('myspider', arg1='val1', start_urls = ["http://example.com"])
 
         def _log(x):
-            print x
+            print(x)
 
         self.spider.log = _log
         self.response = HtmlResponse(body="<html></html>", url="http://www.example.com/product/8798732")

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+six>=1.5.2`
`1`	`2`	`boto`
`2`	`3`	`hubstorage`
`3`	`4`	`python-dateutil`