您好,登錄后才能下訂單哦!
import requests
from lxml import etree
import re
import time
from multiprocessing import Pool #導(dǎo)入multiprocessing庫(kù)的Pool模塊
headers = {'user-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36'}
def get_info(url):
html = requests.get(url,headers = headers)
selector = etree.HTML(html.text)
names = selector.xpath('//*[@class="article block untagged mb15 typs_hot"]/div[1]/a[2]/h3/text()')
centents = re.findall('<div class="content">.*?<span>(.*?)</span>',html.text,re.S) #第一個(gè)正則是為了匹配換行符
laughs = re.findall('<span class="stats-vote"><i class="number">(\d+)</i>',html.text,re.S)
comments = re.findall('<i class="number">(\d+)</i> 評(píng)論',html.text,re.S)
for name,centent,laugh,comment in zip(names,centents,laughs,comments):
info = {
'name':name,
'centent':centents,
'laugh':laughs,
'comment':comments
}
return (info)
if __name__ == '__main__':
urls = ["https://www.qiushibaike.com/text/page/{}/".format(num)for num in range(0,14)]
start_1 = time.time()
for url in urls:
get_info(url)
end_1 = time.time()
print('串行爬取花費(fèi)時(shí)間:' + str(end_1 - start_1))
start_2 = time.time()
pool = Pool(processes=2) #創(chuàng)建進(jìn)程池,processes為設(shè)置的進(jìn)程個(gè)數(shù)
pool.map(get_info,urls) #利用map()函數(shù)運(yùn)行進(jìn)程,參數(shù)fuc為運(yùn)行的函數(shù),iterable為迭代參數(shù)
end_2 = time.time()
print('2個(gè)進(jìn)程:' + str(end_2 - start_2))
start_3 = time.time()
pool = Pool(processes=4) # 創(chuàng)建進(jìn)程池,processes為設(shè)置的進(jìn)程個(gè)數(shù)
pool.map(get_info, urls) # 利用map()函數(shù)運(yùn)行進(jìn)程,參數(shù)fuc為運(yùn)行的函數(shù),iterable為迭代參數(shù)
end_3 = time.time()
print('4個(gè)進(jìn)程:' + str(end_3 - start_3))
輸出:
D:\Python\venv\Scripts\python.exe D:/Python/venv/test12.py
串行爬取花費(fèi)時(shí)間:5.043288469314575
2個(gè)進(jìn)程:3.351191759109497
4個(gè)進(jìn)程:2.882164716720581
Process finished with exit code 0
免責(zé)聲明:本站發(fā)布的內(nèi)容(圖片、視頻和文字)以原創(chuàng)、轉(zhuǎn)載和分享為主,文章觀點(diǎn)不代表本網(wǎng)站立場(chǎng),如果涉及侵權(quán)請(qǐng)聯(lián)系站長(zhǎng)郵箱:is@yisu.com進(jìn)行舉報(bào),并提供相關(guān)證據(jù),一經(jīng)查實(shí),將立刻刪除涉嫌侵權(quán)內(nèi)容。