python如何提升爬蟲效率

import asyncio
import time

start_time = time.time()
async def get_request(url):
  await asyncio.sleep(2)
  print(url,'下載完成!')

urls = [
  'www.1.com',
  'www.2.com',
]

task_lst = [] # 任務(wù)對象列表
for url in urls:
  c = get_request(url) # 協(xié)程對象
  task = asyncio.ensure_future(c) # 任務(wù)對象
  # task.add_done_callback(...)  # 綁定回調(diào)
  task_lst.append(task)

loop = asyncio.get_event_loop() # 事件循環(huán)對象
loop.run_until_complete(asyncio.wait(task_lst)) # 注冊,手動掛起

# 線程池
import time
from multiprocessing.dummy import Pool

start_time = time.time()
url_list = [
  'www.1.com',
  'www.2.com',
  'www.3.com',
]
def get_request(url):
  print('正在下載...',url)
  time.sleep(2)
  print('下載完成!',url)

pool = Pool(3)
pool.map(get_request,url_list)
print('總耗時:',time.time()-start_time)

from flask import Flask
import time

app = Flask(__name__)

@app.route('/bobo')
def index_bobo():
  time.sleep(2)
  return 'hello bobo!'

@app.route('/jay')
def index_jay():
  time.sleep(2)
  return 'hello jay!'

@app.route('/tom')
def index_tom():
  time.sleep(2)
  return 'hello tom!'

if __name__ == '__main__':
  app.run(threaded=True)

import asyncio
import aiohttp
import requests
import time

start = time.time()
async def get_page(url):
  # page_text = requests.get(url=url).text
  # print(page_text)
  # return page_text
  async with aiohttp.ClientSession() as s: #生成一個session對象
    async with await s.get(url=url) as response:
      page_text = await response.text()
      print(page_text)
  return page_text

urls = [
  'http://127.0.0.1:5000/bobo',
  'http://127.0.0.1:5000/jay',
  'http://127.0.0.1:5000/tom',
]
tasks = []
for url in urls:
  c = get_page(url)
  task = asyncio.ensure_future(c)
  tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))

end = time.time()
print(end-start)

# 異步執(zhí)行!
# hello tom!
# hello bobo!
# hello jay!
# 2.0311079025268555

'''
aiohttp模塊實現(xiàn)單線程+多任務(wù)異步協(xié)程
并用xpath解析數(shù)據(jù)
'''
import aiohttp
import asyncio
from lxml import etree
import time

start = time.time()
# 特殊函數(shù):請求的發(fā)送和數(shù)據(jù)的捕獲
# 注意async with await關(guān)鍵字
async def get_request(url):
  async with aiohttp.ClientSession() as s:
    async with await s.get(url=url) as response:
      page_text = await response.text()
      return page_text    # 返回頁面源碼

# 回調(diào)函數(shù),解析數(shù)據(jù)
def parse(task):
  page_text = task.result()
  tree = etree.HTML(page_text)
  msg = tree.xpath('/html/body/ul//text()')
  print(msg)

urls = [
  'http://127.0.0.1:5000/bobo',
  'http://127.0.0.1:5000/jay',
  'http://127.0.0.1:5000/tom',
]
tasks = []
for url in urls:
  c = get_request(url)
  task = asyncio.ensure_future(c)
  task.add_done_callback(parse) #綁定回調(diào)函數(shù)!
  tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))

end = time.time()
print(end-start)

import time
import requests
from multiprocessing.dummy import Pool

start = time.time()
urls = [
  'http://127.0.0.1:5000/bobo',
  'http://127.0.0.1:5000/jay',
  'http://127.0.0.1:5000/tom',
]
def get_request(url):
  page_text = requests.get(url=url).text
  print(page_text)
  return page_text

pool = Pool(3)
pool.map(get_request, urls)
end = time.time()
print('總耗時:', end-start)

# 實現(xiàn)異步請求
# hello jay!
# hello bobo!
# hello tom!
# 總耗時: 2.0467123985290527