Merge branch 'nltk-compat' into dev

Author: sloria

.travis.yml

@@ -5,9 +5,10 @@ python:
     - "2.7"
     - "3.3"
     - "3.4"
+    - "3.5"
 before_install:
-    - "wget https://s3.amazonaws.com/textblob/nltk_data.tar.gz"
-    - "tar -xzvf nltk_data.tar.gz -C ~"
+    - "wget https://s3.amazonaws.com/textblob/nltk_data-0.11.0.tar.gz"
+    - "tar -xzvf nltk_data-0.11.0.tar.gz -C ~"
 # Install dependencies
 install:
     - "pip install numpy"

CHANGELOG.rst

@@ -1,6 +1,19 @@
 Changelog
 =========
 
+0.11.0 (unreleased)
+-------------------
+
+Changes:
+
+- Compatible with nltk>=3.1. NLTK versions < 3.1 are no longer supported.
+- Change default tagger to NLTKTagger (uses NLTK's averaged perceptron tagger).
+- Tested on Python 3.5.
+
+Bug fixes:
+
+- Fix spelling correction when nltk>=3.1 is installed (:issue:`99`). Thanks :user:`shubham12101` for reporting.
+
 0.10.0 (2015-10-04)
 -------------------
 

setup.py

@@ -3,7 +3,7 @@
 import re
 from setuptools import setup, find_packages
 
-REQUIREMENTS = ['nltk>=3.0']
+REQUIREMENTS = ['nltk>=3.1']
 TEST_REQUIREMENTS = ['nose', 'mock']
 
 
@@ -57,6 +57,7 @@ def read(fname):
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         "Topic :: Text Processing :: Linguistic",

tests/test_blob.py

@@ -404,11 +404,11 @@ def test_words(self):
             'better',
             'than',
             'implicit',
-            ]))
+        ]))
         short = tb.TextBlob("Just a bundle of words")
         assert_equal(short.words, tb.WordList([
             'Just', 'a', 'bundle', 'of', 'words'
-            ]))
+        ]))
 
     def test_words_includes_apostrophes_in_contractions(self):
         blob = tb.TextBlob("Let's test this.")
@@ -421,7 +421,7 @@ def test_pos_tags(self):
         blob = tb.TextBlob('Simple is better than complex. '
                             'Complex is better than complicated.')
         assert_equal(blob.pos_tags, [
-            ('Simple', 'JJ'),
+            ('Simple', 'NN'),
             ('is', 'VBZ'),
             ('better', 'JJR'),
             ('than', 'IN'),
@@ -431,7 +431,7 @@ def test_pos_tags(self):
             ('better', 'JJR'),
             ('than', 'IN'),
             ('complicated', 'VBN'),
-            ])
+        ])
 
     def test_tags(self):
         assert_equal(self.blob.tags, self.blob.pos_tags)
@@ -442,7 +442,6 @@ def test_tagging_nonascii(self):
         tags = b.tags
         assert_true(isinstance(tags[0][0], unicode))
 
-
     def test_pos_tags_includes_one_letter_articles(self):
         blob = tb.TextBlob("This is a sentence.")
         assert_equal(blob.pos_tags[2][0], 'a')
@@ -483,14 +482,13 @@ def test_can_get_subjectivity_and_polarity_with_different_analyzer(self):
 
     def test_pos_tagger_defaults_to_pattern(self):
         blob = tb.TextBlob("some text")
-        assert_true(isinstance(blob.pos_tagger, PatternTagger))
+        assert_true(isinstance(blob.pos_tagger, NLTKTagger))
 
     def test_pos_tagger_is_shared_among_instances(self):
         blob1 = tb.TextBlob("This is one sentence")
         blob2 = tb.TextBlob("This is another sentence.")
         assert_true(blob1.pos_tagger is blob2.pos_tagger)
 
-
     def test_can_use_different_pos_tagger(self):
         tagger = NLTKTagger()
         blob = tb.TextBlob("this is some text", pos_tagger=tagger)
@@ -782,6 +780,13 @@ def test_correct(self):
         assert_equal(blob3.correct(), "The meaning of life is 42.0.")
         blob4 = tb.TextBlob("?")
         assert_equal(blob4.correct(), "?")
+
+        blob5 = tb.TextBlob("I can't spel")
+        assert_equal(blob5.correct(), "I can't spell")
+
+        blob6 = tb.TextBlob("I cann't \nspel")
+        assert_equal(blob6.correct(), "I can't \nspell")
+
         # From a user-submitted bug
         text = "Before you embark on any of this journey, write a quick " + \
                 "high-level test that demonstrates the slowness. " + \
@@ -965,7 +970,7 @@ def test_creates_blobs(self):
 
     def test_default_tagger(self):
         blob = self.blobber("Some text")
-        assert_true(isinstance(blob.pos_tagger, PatternTagger))
+        assert_true(isinstance(blob.pos_tagger, NLTKTagger))
 
     def test_default_np_extractor(self):
         blob = self.blobber("Some text")
@@ -976,7 +981,7 @@ def test_default_tokenizer(self):
         assert_true(isinstance(blob.tokenizer, WordTokenizer))
 
     def test_str_and_repr(self):
-        expected = "Blobber(tokenizer=WordTokenizer(), pos_tagger=PatternTagger(), np_extractor=FastNPExtractor(), analyzer=PatternAnalyzer(), parser=PatternParser(), classifier=None)"
+        expected = "Blobber(tokenizer=WordTokenizer(), pos_tagger=NLTKTagger(), np_extractor=FastNPExtractor(), analyzer=PatternAnalyzer(), parser=PatternParser(), classifier=None)"
         assert_equal(repr(self.blobber), expected)
         assert_equal(str(self.blobber), repr(self.blobber))
 

tests/test_taggers.py

@@ -45,11 +45,11 @@ def setUp(self):
     def test_tag(self):
         tags = self.tagger.tag(self.text)
         assert_equal(tags,
-            [('Simple', 'NNP'), ('is', 'VBZ'),
+            [('Simple', 'NN'), ('is', 'VBZ'),
             ('better', 'JJR'), ('than', 'IN'),
             ('complex', 'JJ'), ('.', '.'), ('Complex', 'NNP'),
             ('is', 'VBZ'), ('better', 'JJR'),
-            ('than', 'IN'), ('complicated', 'JJ'), ('.', '.')])
+            ('than', 'IN'), ('complicated', 'VBN'), ('.', '.')])
 
 
 def test_cannot_instantiate_incomplete_tagger():

textblob/blob.py

@@ -35,7 +35,7 @@
 from textblob.base import (BaseNPExtractor, BaseTagger, BaseTokenizer,
                        BaseSentimentAnalyzer, BaseParser)
 from textblob.np_extractors import FastNPExtractor
-from textblob.taggers import PatternTagger
+from textblob.taggers import NLTKTagger
 from textblob.tokenizers import WordTokenizer, sent_tokenize, word_tokenize
 from textblob.sentiments import PatternAnalyzer
 from textblob.parsers import PatternParser
@@ -321,7 +321,7 @@ class BaseBlob(StringlikeMixin, BlobComparableMixin):
     :param np_extractor: (optional) An NPExtractor instance. If ``None``,
         defaults to :class:`FastNPExtractor() <textblob.en.np_extractors.FastNPExtractor>`.
     :param pos_tagger: (optional) A Tagger instance. If ``None``,
-        defaults to :class:`PatternTagger <textblob.en.taggers.PatternTagger>`.
+        defaults to :class:`NLTKTagger <textblob.en.taggers.NLTKTagger>`.
     :param analyzer: (optional) A sentiment analyzer. If ``None``,
         defaults to :class:`PatternAnalyzer <textblob.en.sentiments.PatternAnalyzer>`.
     :param parser: A parser. If ``None``, defaults to
@@ -332,7 +332,7 @@ class BaseBlob(StringlikeMixin, BlobComparableMixin):
         ``clean_html`` parameter deprecated, as it was in NLTK.
     """
     np_extractor = FastNPExtractor()
-    pos_tagger = PatternTagger()
+    pos_tagger = NLTKTagger()
     tokenizer = WordTokenizer()
     translator = Translator()
     analyzer = PatternAnalyzer()
@@ -536,8 +536,8 @@ def correct(self):
 
         :rtype: :class:`BaseBlob <BaseBlob>`
         """
-        # regex matches: contraction or word or punctuation or whitespace
-        tokens = nltk.tokenize.regexp_tokenize(self.raw, "\w*('\w*)+|\w+|[^\w\s]|\s")
+        # regex matches: word or punctuation or whitespace
+        tokens = nltk.tokenize.regexp_tokenize(self.raw, "\w+|[^\w\s]|\s")
         corrected = (Word(w).correct() for w in tokens)
         ret = ''.join(corrected)
         return self.__class__(ret)
@@ -589,7 +589,7 @@ class TextBlob(BaseBlob):
     :param np_extractor: (optional) An NPExtractor instance. If ``None``,
         defaults to :class:`FastNPExtractor() <textblob.en.np_extractors.FastNPExtractor>`.
     :param pos_tagger: (optional) A Tagger instance. If ``None``, defaults to
-        :class:`PatternTagger <textblob.en.taggers.PatternTagger>`.
+        :class:`NLTKTagger <textblob.en.taggers.NLTKTagger>`.
     :param analyzer: (optional) A sentiment analyzer. If ``None``, defaults to
         :class:`PatternAnalyzer <textblob.en.sentiments.PatternAnalyzer>`.
     :param classifier: (optional) A classifier.
@@ -711,7 +711,7 @@ class Blobber(object):
     :param np_extractor: (optional) An NPExtractor instance. If ``None``,
         defaults to :class:`FastNPExtractor() <textblob.en.np_extractors.FastNPExtractor>`.
     :param pos_tagger: (optional) A Tagger instance. If ``None``,
-        defaults to :class:`PatternTagger <textblob.en.taggers.PatternTagger>`.
+        defaults to :class:`NLTKTagger <textblob.en.taggers.NLTKTagger>`.
     :param analyzer: (optional) A sentiment analyzer. If ``None``,
         defaults to :class:`PatternAnalyzer <textblob.en.sentiments.PatternAnalyzer>`.
     :param parser: A parser. If ``None``, defaults to
@@ -722,7 +722,7 @@ class Blobber(object):
     """
 
     np_extractor = FastNPExtractor()
-    pos_tagger = PatternTagger()
+    pos_tagger = NLTKTagger()
     tokenizer = WordTokenizer()
     analyzer = PatternAnalyzer()
     parser = PatternParser()

textblob/download_corpora.py

@@ -18,12 +18,12 @@
 MIN_CORPORA = [
     'brown',  # Required for FastNPExtractor
     'punkt',  # Required for WordTokenizer
-    'wordnet' # Required for lemmatization
+    'wordnet',  # Required for lemmatization
+    'averaged_perceptron_tagger',  # Required for NLTKTagger
 ]
 
 ADDITIONAL_CORPORA = [
     'conll2000',  # Required for ConllExtractor
-    'maxent_treebank_pos_tagger',  # Required for NLTKTagger
     'movie_reviews',  # Required for NaiveBayesAnalyzer
 ]