基于python框架Scrapy爬取自己的博客內(nèi)容過程詳解

scrapy startproject scrapy_cnblogs

scrapy_cnblogs
 ├── botcnblogs
 │ 　　├── __init__.py
 │ 　　├── items.py  #用于定義抓取內(nèi)容的實(shí)體
 │ 　　├── pipelines.py #處理抓取的item的管道
 │ 　　├── settings.py #爬蟲需要的配置參數(shù)在這里
 │ └── spiders
 │ 　　 └── __init__.py
 └── scrapy.cfg   #項目的配置文件，可以不去理會，默認(rèn)即可

[settings]
default = botcnblogs.settings
[deploy]
#url = http://localhost:6800/
project = botcnblogs

from scrapy import Item,Field #引入Item、Field
class BotcnblogsItem(Item):
 # define the fields for your item here like:
 title = Field()  #標(biāo)題
 publishDate = Field() #發(fā)布日期
 readCount = Field() #閱讀量
 commentCount = Field() #評論數(shù)<br data-filtered="filtered"><br data-filtered="filtered">

# -*- coding: utf-8 -*- 
import json 
class BotcnblogsPipeline(object):
 def __init__(self):
  self.file = open("item.json", "w+") 
 def process_item(self, item, spider):
  record = json.dumps(dict(item), ensure_ascii=False)+"\n" #此處如果有中文的話，要加上ensure_ascii=False參數(shù)，否則可能出現(xiàn)亂碼
  self.file.write(record)
  return item 
 def open_spider(self, spider):
  pass 
 def close_spider(self, spider):
  self.file.close()

SPIDER_MODULES = ['botcnblogs.spiders']
NEWSPIDER_MODULE = 'botcnblogs.spiders'
ITEM_PIPELINES = {
 'botcnblogs.pipelines.BotcnblogsPipeline': 1,
}

#-*- coding:utf-8 -*-
__author__ = 'linuxfengzheng'
from scrapy.spiders import Spider, Rule
from scrapy.selector import Selector
from botcnblogs.items import BotcnblogsItem
from scrapy.linkextractors import LinkExtractor
import re
from scrapy.spiders import CrawlSpider 
class botspider(CrawlSpider):
 name = "cnblogsSpider" #設(shè)置爬蟲名稱 
 allowed_domains = ["cnblogs.com"] #設(shè)置允許的域名
 start_urls = [
  "http://www.cnblogs.com/fengzheng/default.html?page=3", #設(shè)置開始爬取頁面
 ] 
 rules = (
  Rule(LinkExtractor(allow=('fengzheng/default.html\?page\=([\d]+)', ),),callback='parse_item',follow=True),
 ) #制定規(guī)則
  def parse_item(self, response):
  sel = response.selector
  posts = sel.xpath('//div[@id="mainContent"]/div/div[@class="day"]')
  items = []
  for p in posts:
   #content = p.extract()
   #self.file.write(content.encode("utf-8"))
   item = BotcnblogsItem()
   publishDate = p.xpath('div[@class="dayTitle"]/a/text()').extract_first()
 
   item["publishDate"] = (publishDate is not None and [publishDate.encode("utf-8")] or [""])[0]
   #self.file.write(title.encode("utf-8"))
   title = p.xpath('div[@class="postTitle"]/a/text()').extract_first()
   item["title"] = (title is not None and [title.encode("utf-8")] or [""])[0]
 
   #re_first("posted @ 2015-11-03 10:32 風(fēng)的姿態(tài) 閱讀(\d+")
 
   readcount = p.xpath('div[@class="postDesc"]/text()').re_first(u"閱讀\(\d+\)")
 
   regReadCount = re.search(r"\d+", readcount)
   if regReadCount is not None:
    readcount = regReadCount.group()
   item["readCount"] = (readcount is not None and [readcount.encode("utf-8")] or [0])[0]
 
   commentcount = p.xpath('div[@class="postDesc"]/text()').re_first(u"評論\(\d+\)")
   regCommentCount = re.search(r"\d+", commentcount)
   if regCommentCount is not None:
    commentcount = regCommentCount.group()
   item["commentCount"] = (commentcount is not None and [commentcount.encode("utf-8")] or [0])[0]
   items.append(item) 
  return items
  #self.file.close()

start_urls = [
  "http://www.cnblogs.com/fengzheng/default.html?page=1",
  "http://www.cnblogs.com/fengzheng/default.html?page=2",
  "http://www.cnblogs.com/fengzheng/default.html?page=3",
 ]

rules = (
  Rule(LinkExtractor(allow=('fengzheng/default.html\?page\=([\d]+)', ),),callback='parse_item',follow=True),
 )

posts = sel.xpath('//div[@id="mainContent"]/div/div[@class="day"]')

scrapy crawl cnblogsSpider