More scrapy

Author: REMitchell

v2/Chapter05_Scrapy/wikiSpider/wikiSpider/article.py

@@ -11,5 +11,8 @@ def start_requests(self):
         return [scrapy.Request(url=url, callback=self.parse) for url in urls]
 
     def parse(self, response):
+        url = response.url
         title = response.css('h1::text').extract_first()
+        print('URL is: {}'.format(url))
         print('Title is: {}'.format(title))
+

v2/Chapter05_Scrapy/wikiSpider/wikiSpider/articleItems.py

@@ -12,6 +12,7 @@ class ArticleSpider(CrawlSpider):
 
     def parse_items(self, response):
         article = Article()
+        article['url'] = response.url
         article['title'] = response.css('h1::text').extract_first()
         article['text'] = response.xpath('//div[@id="mw-content-text"]//text()').extract()
         lastUpdated = response.css('li#footer-info-lastmod::text').extract_first()

v2/Chapter05_Scrapy/wikiSpider/wikiSpider/articles.py

@@ -8,10 +8,12 @@ class ArticleSpider(CrawlSpider):
     rules = [Rule(LinkExtractor(allow=r'.*'), callback='parse_items', follow=True)]
 
     def parse_items(self, response):
+        url = response.url
         title = response.css('h1::text').extract_first()
         text = response.xpath('//div[@id="mw-content-text"]//text()').extract()
         lastUpdated = response.css('li#footer-info-lastmod::text').extract_first()
         lastUpdated = lastUpdated.replace('This page was last edited on ', '')
+        print('URL is: {}'.format(url))
         print('title is: {} '.format(title))
         print('text is: {}'.format(text))
         print('Last updated: {}'.format(lastUpdated))

v2/Chapter05_Scrapy/wikiSpider/wikiSpider/articlesMoreRules.py

@@ -14,6 +14,7 @@ def parse_items(self, response, is_article):
         print(response.url)
         title = response.css('h1::text').extract_first()
         if is_article:
+            url = response.url
             text = response.xpath('//div[@id="mw-content-text"]//text()').extract()
             lastUpdated = response.css('li#footer-info-lastmod::text').extract_first()
             lastUpdated = lastUpdated.replace('This page was last edited on ', '')

v2/Chapter05_Scrapy/wikiSpider/wikiSpider/items.py

@@ -9,7 +9,7 @@
 
 
 class Article(scrapy.Item):
+    url = scrapy.Field()
     title = scrapy.Field()
     text = scrapy.Field()
     lastUpdated = scrapy.Field()
-    pass