Working on Scrapy

Author: REMitchell

v2/Chapter05_Scrapy/wikiSpider/wikiSpider/articles.py

@@ -0,0 +1,15 @@
+import scrapy
+
+class ArticleSpider(scrapy.Spider):
+    name='articles'
+
+    def start_requests(self):
+        urls = [
+            "http://en.wikipedia.org/wiki/Python_%28programming_language%29",
+            "https://en.wikipedia.org/wiki/Functional_programming",
+            "https://en.wikipedia.org/wiki/Monty_Python"]
+        return [scrapy.Request(url=a, callback=self.parse) for a in urls]
+
+    def parse(self, response):
+        title = response.css('h1::text').extract_first()
+        print('Title is: {}'.format(title))

v2/Chapter05_Scrapy/wikiSpider/wikiSpider/items.py

@@ -1,8 +1,14 @@
-from scrapy import Item, Field
+# -*- coding: utf-8 -*-
 
+# Define here the models for your scraped items
+#
+# See documentation in:
+# http://doc.scrapy.org/en/latest/topics/items.html
 
-class Article(Item):
-    title = Field()
-    last_edited = Field()
-    url = Field()
+import scrapy
 
+
+class WikispiderItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass

v2/Chapter05_Scrapy/wikiSpider/wikiSpider/settings.py

@@ -13,7 +13,7 @@
 
 SPIDER_MODULES = ['wikiSpider.spiders']
 NEWSPIDER_MODULE = 'wikiSpider.spiders'
-LOG_LEVEL = 'ERROR'
+
 
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 #USER_AGENT = 'wikiSpider (+http://www.yourdomain.com)'