Python Scrapy爬蟲框架如何使用

scrapy startproject zhuanti_new

from zhuanti_new.items import ZhuantiNewItemimport scrapyfrom scrapy.selector import Selector
class JianshuSpiderSpider(scrapy.Spider):   name = 'zhuantispider'   allowed_domains = ['jianshu.com']   start_urls = ['https://www.jianshu.com/recommendations/collections?page=1&order_by=hot']   #print(start_urls)
   def parse(self, response):       '''解析外面頁面'''       selector = Selector(response)       partical_urls = selector.re('<div class="count"><a target="_blank" href="(.*?)">', selector)       for url in partical_urls:           print(url)           right_url = response.urljoin(url)           #print(right_url)           parts = ['?order_by=added_at&page={0}'.format(k) for k in range(1, 11)]           for part in parts:               # 要爬取每個專題的前10個文章的信息               real_url = right_url + part               #print(real_url)               yield scrapy.Request(real_url, callback=self.parse_detail)
       links = ['https://www.jianshu.com/recommendations/collections?page={}&order_by=hot'.format(i) for i in                range(2, 3)]       for link in links:           print(link)           request = scrapy.Request(link, callback=self.parse)           yield request
   def parse_detail(self, response):       selector = Selector(response)       content = selector.xpath('//div[@class="content"]')       for detail in content:           try:               title = detail.xpath('a[1]/text()').extract_first()               summary = detail.xpath('p/text()').extract_first().strip()               author = detail.xpath('div/a[1]/text()').extract_first()               # comments = detail.xpath('div/a[2]/text()').extract_first()這樣是不行的               comments = detail.xpath('div/a[2]/text()').extract()[1].strip()               likes = detail.xpath('div/span[1]/text()').extract_first().strip()               money = detail.xpath('div/span[2]/text()').extract_first()
               item = ZhuantiNewItem()               if money is not None:                   item['title'] = title                   item['summary'] = summary                   item['author'] = author                   item['comments'] = comments                   item['likes'] = likes                   item['money'] = money.strip()               else:                   item['title'] = title                   item['summary'] = summary                   item['author'] = author                   item['comments'] = comments                   item['likes'] = likes               print(item)               yield item           except:               pass