在scrapy中使用selenium實(shí)現(xiàn)一個(gè)爬取網(wǎng)頁的功能

# 文件：E:\Miniconda\Lib\site-packages\scrapy\core\downloader\middleware.py
"""
Downloader Middleware manager

See documentation in docs/topics/downloader-middleware.rst
"""
import six

from twisted.internet import defer

from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.conf import build_component_list


class DownloaderMiddlewareManager(MiddlewareManager):

  component_name = 'downloader middleware'

  @classmethod
  def _get_mwlist_from_settings(cls, settings):
    # 從settings.py或這custom_setting中拿到自定義的Middleware中間件
    '''
    'DOWNLOADER_MIDDLEWARES': {
      'mySpider.middlewares.ProxiesMiddleware': 400,
      # SeleniumMiddleware
      'mySpider.middlewares.SeleniumMiddleware': 543,
      'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
    },
    '''
    return build_component_list(
      settings.getwithbase('DOWNLOADER_MIDDLEWARES'))

  # 將所有自定義Middleware中間件的處理函數(shù)添加到對(duì)應(yīng)的methods列表中
  def _add_middleware(self, mw):
    if hasattr(mw, 'process_request'):
      self.methods['process_request'].append(mw.process_request)
    if hasattr(mw, 'process_response'):
      self.methods['process_response'].insert(0, mw.process_response)
    if hasattr(mw, 'process_exception'):
      self.methods['process_exception'].insert(0, mw.process_exception)

  # 整個(gè)下載流程
  def download(self, download_func, request, spider):
    @defer.inlineCallbacks
    def process_request(request):
      # 處理request請(qǐng)求，依次經(jīng)過各個(gè)自定義Middleware中間件的process_request方法，前面有加入到list中
      for method in self.methods['process_request']:
        response = yield method(request=request, spider=spider)
        assert response is None or isinstance(response, (Response, Request)), \
            'Middleware %s.process_request must return None, Response or Request, got %s' % \
            (six.get_method_self(method).__class__.__name__, response.__class__.__name__)
        # 這是關(guān)鍵地方
        # 如果在某個(gè)Middleware中間件的process_request中處理完之后，生成了一個(gè)response對(duì)象
        # 那么會(huì)直接將這個(gè)response return 出去，跳出循環(huán)，不再處理其他的process_request
        # 之前我們的header，proxy中間件，都只是加個(gè)user-agent，加個(gè)proxy，并不做任何return值
        # 還需要注意一點(diǎn):就是這個(gè)return的必須是Response對(duì)象
        # 后面我們構(gòu)造的HtmlResponse正是Response的子類對(duì)象
        if response:
          defer.returnValue(response)
      # 如果在上面的所有process_request中，都沒有返回任何Response對(duì)象的話
      # 最后，會(huì)將這個(gè)加工過的Request送往download_func，進(jìn)行下載，返回的就是一個(gè)Response對(duì)象
      # 然后依次經(jīng)過各個(gè)Middleware中間件的process_response方法進(jìn)行加工，如下
      defer.returnValue((yield download_func(request=request,spider=spider)))

    @defer.inlineCallbacks
    def process_response(response):
      assert response is not None, 'Received None in process_response'
      if isinstance(response, Request):
        defer.returnValue(response)

      for method in self.methods['process_response']:
        response = yield method(request=request, response=response,
                    spider=spider)
        assert isinstance(response, (Response, Request)), \
          'Middleware %s.process_response must return Response or Request, got %s' % \
          (six.get_method_self(method).__class__.__name__, type(response))
        if isinstance(response, Request):
          defer.returnValue(response)
      defer.returnValue(response)

    @defer.inlineCallbacks
    def process_exception(_failure):
      exception = _failure.value
      for method in self.methods['process_exception']:
        response = yield method(request=request, exception=exception,
                    spider=spider)
        assert response is None or isinstance(response, (Response, Request)), \
          'Middleware %s.process_exception must return None, Response or Request, got %s' % \
          (six.get_method_self(method).__class__.__name__, type(response))
        if response:
          defer.returnValue(response)
      defer.returnValue(_failure)

    deferred = mustbe_deferred(process_request, request)
    deferred.addErrback(process_exception)
    deferred.addCallback(process_response)
    return deferred

# 文件settings.py中

# ----------- selenium參數(shù)配置 -------------
SELENIUM_TIMEOUT = 25      # selenium瀏覽器的超時(shí)時(shí)間，單位秒
LOAD_IMAGE = True        # 是否下載圖片
WINDOW_HEIGHT = 900       # 瀏覽器窗口大小
WINDOW_WIDTH = 900

# 文件mySpider.py中
class mySpider(CrawlSpider):
  name = "mySpiderAmazon"
  allowed_domains = ['amazon.com']

  custom_settings = {
    'LOG_LEVEL':'INFO',
    'DOWNLOAD_DELAY': 0,
    'COOKIES_ENABLED': False, # enabled by default
    'DOWNLOADER_MIDDLEWARES': {
      # 代理中間件
      'mySpider.middlewares.ProxiesMiddleware': 400,
      # SeleniumMiddleware 中間件
      'mySpider.middlewares.SeleniumMiddleware': 543,
      # 將scrapy默認(rèn)的user-agent中間件關(guān)閉
      'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
    },

#.....................華麗的分割線.......................
# 生成request時(shí)，將是否使用selenium下載的標(biāo)記，放入到meta中
yield Request(
  url = "https://www.amazon.com/",
  meta = {'usedSelenium': True, 'dont_redirect': True},
  callback = self.parseIndexPage,
  errback = self.error
)

# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from scrapy.http import HtmlResponse
from logging import getLogger
import time

class SeleniumMiddleware():
  # 經(jīng)常需要在pipeline或者中間件中獲取settings的屬性，可以通過scrapy.crawler.Crawler.settings屬性
  @classmethod
  def from_crawler(cls, crawler):
    # 從settings.py中，提取selenium設(shè)置參數(shù)，初始化類
    return cls(timeout=crawler.settings.get('SELENIUM_TIMEOUT'),
          isLoadImage=crawler.settings.get('LOAD_IMAGE'),
          windowHeight=crawler.settings.get('WINDOW_HEIGHT'),
          windowWidth=crawler.settings.get('WINDOW_WIDTH')
          )

  def __init__(self, timeout=30, isLoadImage=True, windowHeight=None, windowWidth=None):
    self.logger = getLogger(__name__)
    self.timeout = timeout
    self.isLoadImage = isLoadImage
    # 定義一個(gè)屬于這個(gè)類的browser，防止每次請(qǐng)求頁面時(shí)，都會(huì)打開一個(gè)新的chrome瀏覽器
    # 這樣，這個(gè)類處理的Request都可以只用這一個(gè)browser
    self.browser = webdriver.Chrome()
    if windowHeight and windowWidth:
      self.browser.set_window_size(900, 900)
    self.browser.set_page_load_timeout(self.timeout)    # 頁面加載超時(shí)時(shí)間
    self.wait = WebDriverWait(self.browser, 25)       # 指定元素加載超時(shí)時(shí)間

    def process_request(self, request, spider):
    '''
    用chrome抓取頁面
    :param request: Request請(qǐng)求對(duì)象
    :param spider: Spider對(duì)象
    :return: HtmlResponse響應(yīng)
    '''
    # self.logger.debug('chrome is getting page')
    print(f"chrome is getting page")
    # 依靠meta中的標(biāo)記，來決定是否需要使用selenium來爬取
    usedSelenium = request.meta.get('usedSelenium', False)
    if usedSelenium:
      try:
        self.browser.get(request.url)
        # 搜索框是否出現(xiàn)
        input = self.wait.until(
          EC.presence_of_element_located((By.XPATH, "//div[@class='nav-search-field ']/input"))
        )
        time.sleep(2)
        input.clear()
        input.send_keys("iphone 7s")
        # 敲enter鍵, 進(jìn)行搜索
        input.send_keys(Keys.RETURN)
        # 查看搜索結(jié)果是否出現(xiàn)
        searchRes = self.wait.until(
          EC.presence_of_element_located((By.XPATH, "//div[@id='resultsCol']"))
        )
      except Exception as e:
        # self.logger.debug(f'chrome getting page error, Exception = {e}')
        print(f"chrome getting page error, Exception = {e}")
        return HtmlResponse(url=request.url, status=500, request=request)
      else:
        time.sleep(3)
        return HtmlResponse(url=request.url,
                  body=self.browser.page_source,
                  request=request,
                  # 最好根據(jù)網(wǎng)頁的具體編碼而定
                  encoding='utf-8',
                  status=200)

# 文件settings.py中

# ----------- selenium參數(shù)配置 -------------
SELENIUM_TIMEOUT = 25      # selenium瀏覽器的超時(shí)時(shí)間，單位秒
LOAD_IMAGE = True        # 是否下載圖片
WINDOW_HEIGHT = 900       # 瀏覽器窗口大小
WINDOW_WIDTH = 900

# 文件mySpider.py中
# selenium相關(guān)庫
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait

# scrapy 信號(hào)相關(guān)庫
from scrapy.utils.project import get_project_settings
# 下面這種方式，即將廢棄，所以不用
# from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
# scrapy最新采用的方案
from pydispatch import dispatcher

class mySpider(CrawlSpider):
  name = "mySpiderAmazon"
  allowed_domains = ['amazon.com']

  custom_settings = {
    'LOG_LEVEL':'INFO',
    'DOWNLOAD_DELAY': 0,
    'COOKIES_ENABLED': False, # enabled by default
    'DOWNLOADER_MIDDLEWARES': {
      # 代理中間件
      'mySpider.middlewares.ProxiesMiddleware': 400,
      # SeleniumMiddleware 中間件
      'mySpider.middlewares.SeleniumMiddleware': 543,
      # 將scrapy默認(rèn)的user-agent中間件關(guān)閉
      'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
    },

  # 將chrome初始化放到spider中，成為spider中的元素
  def __init__(self, timeout=30, isLoadImage=True, windowHeight=None, windowWidth=None):
    # 從settings.py中獲取設(shè)置參數(shù)
    self.mySetting = get_project_settings()
    self.timeout = self.mySetting['SELENIUM_TIMEOUT']
    self.isLoadImage = self.mySetting['LOAD_IMAGE']
    self.windowHeight = self.mySetting['WINDOW_HEIGHT']
    self.windowWidth = self.mySetting['windowWidth']
    # 初始化chrome對(duì)象
    self.browser = webdriver.Chrome()
    if self.windowHeight and self.windowWidth:
      self.browser.set_window_size(900, 900)
    self.browser.set_page_load_timeout(self.timeout)    # 頁面加載超時(shí)時(shí)間
    self.wait = WebDriverWait(self.browser, 25)       # 指定元素加載超時(shí)時(shí)間
    super(mySpider, self).__init__()
    # 設(shè)置信號(hào)量，當(dāng)收到spider_closed信號(hào)時(shí)，調(diào)用mySpiderCloseHandle方法，關(guān)閉chrome
    dispatcher.connect(receiver = self.mySpiderCloseHandle,
              signal = signals.spider_closed
              )

  # 信號(hào)量處理函數(shù)：關(guān)閉chrome瀏覽器
  def mySpiderCloseHandle(self, spider):
    print(f"mySpiderCloseHandle: enter ")
    self.browser.quit()

#.....................華麗的分割線.......................
# 生成request時(shí)，將是否使用selenium下載的標(biāo)記，放入到meta中
yield Request(
  url = "https://www.amazon.com/",
  meta = {'usedSelenium': True, 'dont_redirect': True},
  callback = self.parseIndexPage,
  errback = self.error
)

# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from scrapy.http import HtmlResponse
from logging import getLogger
import time

class SeleniumMiddleware():
  # Middleware中會(huì)傳遞進(jìn)來一個(gè)spider，這就是我們的spider對(duì)象，從中可以獲取__init__時(shí)的chrome相關(guān)元素
  def process_request(self, request, spider):
    '''
    用chrome抓取頁面
    :param request: Request請(qǐng)求對(duì)象
    :param spider: Spider對(duì)象
    :return: HtmlResponse響應(yīng)
    '''
    print(f"chrome is getting page")
    # 依靠meta中的標(biāo)記，來決定是否需要使用selenium來爬取
    usedSelenium = request.meta.get('usedSelenium', False)
    if usedSelenium:
      try:
        spider.browser.get(request.url)
        # 搜索框是否出現(xiàn)
        input = spider.wait.until(
          EC.presence_of_element_located((By.XPATH, "//div[@class='nav-search-field ']/input"))
        )
        time.sleep(2)
        input.clear()
        input.send_keys("iphone 7s")
        # 敲enter鍵, 進(jìn)行搜索
        input.send_keys(Keys.RETURN)
        # 查看搜索結(jié)果是否出現(xiàn)
        searchRes = spider.wait.until(
          EC.presence_of_element_located((By.XPATH, "//div[@id='resultsCol']"))
        )
      except Exception as e:
        print(f"chrome getting page error, Exception = {e}")
        return HtmlResponse(url=request.url, status=500, request=request)
      else:
        time.sleep(3)
        # 頁面爬取成功，構(gòu)造一個(gè)成功的Response對(duì)象(HtmlResponse是它的子類)
        return HtmlResponse(url=request.url,
                  body=spider.browser.page_source,
                  request=request,
                  # 最好根據(jù)網(wǎng)頁的具體編碼而定
                  encoding='utf-8',
                  status=200)