詳解python中urllib爬蟲模塊

from urllib import request
url = 'http://www.baidu.com/'

# 向百度發(fā)請(qǐng)求,得到響應(yīng)對(duì)象
response = request.urlopen(url)

# 返回網(wǎng)頁(yè)源代碼
print(response.read().decode('utf-8'))

# 返回http響應(yīng)碼
print(response.getcode())    # 200
# 返回實(shí)際數(shù)據(jù)URL地址
print(response.geturl())    # http://www.baidu.com/

from urllib import request
url = 'http://httpbin.org/get'
headers = {'User-Agent':'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)'}
# 創(chuàng)建請(qǐng)求對(duì)象(包裝請(qǐng)求)
req = request.Request(url=url,headers=headers)
# 發(fā)請(qǐng)求,獲取響應(yīng)對(duì)象
res = request.urlopen(req)
# 讀取內(nèi)容,返回網(wǎng)頁(yè)代碼
html = res.read().decode('utf-8')
print(html)

from urllib import parse
url = 'http://www.baidu.com/s?'
query_string = parse.urlencode({'wd':'美女'})
print(query_string)     # wd=%E7%BE%8E%E5%A5%B3
url = url + query_string
# http://www.baidu.com/wd=%E7%BE%8E%E5%A5%B3

from urllib import parse
query_string_dict = {'wd' : '美女',
          'pn' : '50'}
query_string = parse.urlencode(query_string_dict)
url = 'http://www.baidu.com/s?{}'.format(query_string)
print(url)
# http://www.baidu.com/s?wd=%E7%BE%8E%E5%A5%B3&pn=50

from urllib import request
from urllib import parse

# 定義常用變量
word = input('請(qǐng)輸入搜索內(nèi)容:')
url = 'http://www.baidu.com/s?'
headers = {'User-Agent':'Mozilla/5.0'}

# url編碼,拼接完整URL
query_string = parse.urlencode({'wd':word})
url = url + query_string

# 三步走
req = request.Request(url=url,headers=headers)
res = request.urlopen(req)
html = res.read().decode('utf-8')

filename = '{}.html'.format(word)
with open(filename,'w',encoding='utf-8') as f:
  f.write(html)

from urllib import parse
parse.quote('美女')    # %E7%BE%8E%E5%A5%B3

from urllib import parse
result = parse.unquote('%E7%BE%8E%E5%A5%B3')
print(result)    # 美女

from urllib import request,parse
import time
import random

class BaiduSpider(object):
  def __init__(self):
    self.url = 'http://tieba.baidu.com/f?kw={}&pn={}'
    self.headers = {'User-Agent':'Mozilla/5.0'}

  # 獲取響應(yīng)
  def get_page(self,url):
    req = request.Request(url=url,headers=self.headers)
    res = request.urlopen(req)
    html = res.read().decode('utf-8')
    return html

  # 保存數(shù)據(jù)
  def write_page(self,filename,html):
    with open(filename,'w') as f:
      f.write(html)

  # 主函數(shù)
  def main(self):
    name = input('請(qǐng)輸入貼吧名:')
    start = int(input('請(qǐng)輸入起始頁(yè):'))
    end = int(input('請(qǐng)輸入終止頁(yè):'))

    # 拼接URL地址,發(fā)請(qǐng)求
    for page in range(start,end+1):
      pn = (page-1)*50
      kw = parse.quote(name)     # url編碼
      url = self.url.format(kw,pn)
      html = self.get_page(url)    # 獲取響應(yīng),并保存
      filename = '{}-第{}頁(yè).html'.format(name,page)
      self.write_page(filename,html)
      print('第{}頁(yè)爬取成功'.format(page)) # 提示進(jìn)度
      time.sleep(random.randint(1,3))   # 控制爬取速度

if __name__ == '__main__':
  spider = BaiduSpider()
  spider.main()