python爬蟲(chóng)開(kāi)發(fā)之使用python爬蟲(chóng)庫(kù)requests，urllib與今日頭條搜索功能爬取搜索內(nèi)容實(shí)例

import re,json,requests,os
from hashlib import md5
from urllib.parse import urlencode
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
from multiprocessing import Pool
#請(qǐng)求索引頁(yè)
def get_page_index(offset,keyword):
  #傳送的數(shù)據(jù)
  data={
    'offset': offset,
    'format': 'json',
    'keyword': keyword,
    'autoload': 'true',
    'count': '20',
    'cur_tab': 1
  }
  #自動(dòng)編碼為服務(wù)器可識(shí)別的url
  url="https://www.toutiao.com/search_content/?"+urlencode(data)
  #異常處理
  try:
    #獲取返回的網(wǎng)頁(yè)
    response=requests.get(url)
    #判斷網(wǎng)頁(yè)的狀態(tài)碼是否正常獲取
    if response.status_code==200:
      #返回解碼后的網(wǎng)頁(yè)
      return response.text
    #不正常獲取，返回None
    return None
  except RequestException:
    #提示信息
    print("請(qǐng)求索引頁(yè)出錯(cuò)")
    return None
#解析請(qǐng)求的索引網(wǎng)頁(yè)數(shù)據(jù)
def parse_page_index(html):
  #json加載轉(zhuǎn)換
  data=json.loads(html)
  #數(shù)據(jù)為真，并且data鍵值存在與數(shù)據(jù)中
  if data and 'data' in data.keys():
    #遍歷返回圖集所在的url
    for item in data.get('data'):
      yield item.get('article_url')
#圖集詳情頁(yè)請(qǐng)求
def get_page_detail(url):
  #設(shè)置UA，模擬瀏覽器正常訪問(wèn)
  head = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}
  #異常處理
  try:
    response=requests.get(url,headers=head)
    if response.status_code==200:
      return response.text
    return None
  except RequestException:
    print("請(qǐng)求詳情頁(yè)出錯(cuò)")
    return None
#解析圖集詳情頁(yè)的數(shù)據(jù)
def parse_page_detail(html,url):
  #異常處理
  try:
    #格式轉(zhuǎn)換與圖集標(biāo)題提取
    soup=BeautifulSoup(html,'lxml')
    title=soup.select('title')[0].get_text()
    print(title)
    #正則查找圖集鏈接
    image_pattern = re.compile('gallery: (.*?),\n', re.S)
    result = re.search(image_pattern, html)
    if result:
      #數(shù)據(jù)的優(yōu)化
      result=result.group(1)
      result = result[12:]
      result = result[:-2]
      #替換
      result = re.sub(r'\\', '', result)
      #json加載
      data = json.loads(result)
      #判斷數(shù)據(jù)不為空，并確保sub——images在其中
      if data and 'sub_images' in data.keys():
        #sub_images數(shù)據(jù)提取
        sub_images=data.get('sub_images')
        #列表數(shù)據(jù)提取
        images=[item.get('url') for item in sub_images]
        #圖片下載
        for image in images:download_images(image)
        #返回字典
        return {
          'title':title,
          'url':url,
          'images':images
        }
  except Exception:
    pass
#圖片url請(qǐng)求
def download_images(url):
  #提示信息
  print('正在下載',url)
  #瀏覽器模擬
  head = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}
  #異常處理
  try:
    response = requests.get(url, headers=head)
    if response.status_code == 200:
      #圖片保存
      save_image(response.content)
    return None
  except RequestException:
    print("請(qǐng)求圖片出錯(cuò)")
    return None
#圖片保存
def save_image(content):
  #判斷文件夾是否存在，不存在則創(chuàng)建
  if '街拍' not in os.listdir():
    os.makedirs('街拍')
  #設(shè)置寫(xiě)入文件所在文件夾位置
  os.chdir('E:\python寫(xiě)網(wǎng)路爬蟲(chóng)\CSDN爬蟲(chóng)學(xué)習(xí)\街拍')
  #路徑，名稱，后綴
  file_path='{0}/{1}.{2}'.format(os.getcwd(),md5(content).hexdigest(),'jpg')
  #圖片保存
  with open(file_path,'wb') as f:
      f.write(content)
      f.close()
#主函數(shù)
def mian(offset):
  #網(wǎng)頁(yè)獲取
  html=get_page_index(offset,'街拍')
  #圖集url
  for url in parse_page_index(html):
    if url!=None:
      #圖集網(wǎng)頁(yè)詳情
      html=get_page_detail(url)
      #圖集內(nèi)容
      result=parse_page_detail(html,url)
if __name__ == '__main__':
  #創(chuàng)建訪問(wèn)的列表（0-9）頁(yè)
  group=[i*10 for i in range(10)]
  #創(chuàng)建多線程進(jìn)程池
  pool=Pool()
  #進(jìn)程池啟動(dòng)，傳入的數(shù)據(jù)
  pool.map(mian,group)