Feature: download avatar pictures.

Author: songzy12

weibo_spider/downloader/__init__.py

@@ -1,5 +1,9 @@
 from .origin_picture_downloader import OriginPictureDownloader
 from .retweet_picture_downloader import RetweetPictureDownloader
+from .avatar_picture_downloader import AvatarPictureDownloader
 from .video_downloader import VideoDownloader
 
-__all__ = [OriginPictureDownloader, RetweetPictureDownloader, VideoDownloader]
+__all__ = [
+    OriginPictureDownloader, RetweetPictureDownloader, AvatarPictureDownloader,
+    VideoDownloader
+]

weibo_spider/downloader/avatar_picture_downloader.py

@@ -0,0 +1,22 @@
+import os
+
+from .img_downloader import ImgDownloader
+
+
+class AvatarPictureDownloader(ImgDownloader):
+    def __init__(self, file_dir, file_download_timeout):
+        super().__init__(file_dir, file_download_timeout)
+        self.describe = u'头像图片'
+        self.key = 'avatar_pictures'
+
+    def handle_download(self, urls):
+        """处理下载相关操作"""
+        file_dir = self.file_dir + os.sep + self.describe
+        if not os.path.isdir(file_dir):
+            os.makedirs(file_dir)
+
+        for i, url in enumerate(urls):
+            index = url.rfind('/')
+            file_name = url[index:]
+            file_path = file_dir + os.sep + file_name
+            self.download_one_file(url, file_path, 'xxx')

weibo_spider/parser/__init__.py

@@ -1,4 +1,6 @@
 from .index_parser import IndexParser
 from .page_parser import PageParser
+from .photo_parser import PhotoParser
+from .album_parser import AlbumParser
 
-__all__ = [IndexParser, PageParser]
+__all__ = [IndexParser, PageParser, PhotoParser, AlbumParser]

weibo_spider/spider.py

@@ -15,7 +15,8 @@
 from tqdm import tqdm
 
 from . import config_util, datetime_util
-from .parser import IndexParser, PageParser
+from .downloader import AvatarPictureDownloader
+from .parser import IndexParser, PageParser, PhotoParser, AlbumParser
 from .user import User
 
 FLAGS = flags.FLAGS
@@ -140,6 +141,16 @@ def get_user_info(self, user_uri):
         self.user = IndexParser(self.cookie, user_uri).get_user()
         self.page_count += 1
 
+    def download_user_avatar(self, user_uri):
+        """下载用户头像"""
+        avatar_album_url = PhotoParser(self.cookie,
+                                       user_uri).extract_avatar_album_url()
+        pic_urls = AlbumParser(self.cookie,
+                               avatar_album_url).extract_pic_urls()
+        AvatarPictureDownloader(
+            self._get_filepath('img'),
+            self.file_download_timeout).handle_download(pic_urls)
+
     def get_weibo_info(self):
         """获取微博信息"""
         try:
@@ -302,6 +313,10 @@ def get_one_user(self, user_config):
             self.write_user(self.user)
             logger.info('*' * 100)
 
+            # 下载用户头像相册中的图片。
+            if self.pic_download:
+                self.download_user_avatar(user_config['user_uri'])
+
             for weibos in self.get_weibo_info():
                 self.write_weibo(weibos)
                 self.got_num += len(weibos)