您好,登錄后才能下訂單哦!
小編給大家分享一下Python中微信爬蟲的示例分析,相信大部分人都還不怎么了解,因此分享這篇文章給大家參考一下,希望大家閱讀完這篇文章后大有收獲,下面讓我們一起去了解一下吧!
單線程版:
import urllib.request import urllib.parse import urllib.error import re,time headers = ("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3107.4 Safari/537.36") operner = urllib.request.build_opener() operner.addheaders = [headers] urllib.request.install_opener(operner) list_url = [] ###使用代理獲取網(wǎng)頁url內(nèi)容 def use_proxy(url): try: # proxy = urllib.request.ProxyHandler({'http':proxy_addr}) ##使用代理版 # operner = urllib.request.build_opener() # urllib.request.install_opener(operner) headers = ("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3107.4 Safari/537.36") operner = urllib.request.build_opener() operner.addheaders = [headers] urllib.request.install_opener(operner) data = urllib.request.urlopen(url).read().decode('utf-8') # print (data) return data except urllib.error.URLError as e: if hasattr(e, "code"): print(e.code) elif hasattr(e, "reason"): print(e.reason) except Exception as e: print("exception" + str(e)) time.sleep(1) ##獲取要爬取的url def get_url(key, pagestart, pageend): try: keycode = urllib.parse.quote(key) for page in range(pagestart, pageend + 1): url = "http://weixin.sogou.com/weixin?query=%s&_sug_type_=&s_from=input&_sug_=n&type=%d&page=1&ie=utf8" % ( keycode, page) data1 = use_proxy(url) #print("data1的內(nèi)容是", data1) listurl_pattern = '<h4>.*?("http://.*?)</h4>' result = re.compile(listurl_pattern, re.S).findall(data1) for i in range(len(result)): res = result[i].replace("amp;", "").split(" ")[0].replace("\"", "") list_url.append(res) #print(list_url) return list_url except urllib.error.URLError as e: if hasattr(e, "code"): print(e.code) elif hasattr(e, "reason"): print(e.reason) except Exception as e: print("exception:", e) ##通過獲取的url爬行內(nèi)容數(shù)據(jù)并處理 def get_url_content(list_url): fh2=open("D:\\python-script\\1.html", 'wb') html1 = '''<!DOCTYPE html>\n<html xmlns="http://www.w3.org/1999/xhmtl">\n<head>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n<title>微信文章</title></head>\n<body>''' fh2.write(html1.encode("utf-8")) fh2.close() fh = open("D:\\python-script\\1.html", 'ab') for url in list_url: data_content = use_proxy(url) #print (data_content) #sys.exit() title_pattern = '<h3.*>.*?</h3>' result_title = re.compile(title_pattern, re.S).findall(data_content) ##標(biāo)題(str) res_title = result_title[0].replace("<h3 class=\"rich_media_title\" id=\"activity-name\">", "").replace("</h3>", "").strip() content_pattern = 'id="js_content">(.*?)<div class="rich_media_tool" id="js_sg_bar">' content = re.compile(content_pattern, re.S).findall(data_content) try: fh.write(res_title.encode("utf-8")) for i in content: fh.write(i.strip().encode("utf-8")) except UnicodeEncodeError as e: continue fh.write("</body></html>".encode("utf-8")) if __name__ == '__main__': pagestart = 1 pageend = 2 key = "人工智能" get_url(key, pagestart, pageend) get_url_content(list_url)
多線程版:
import urllib.request import urllib.parse import urllib.error import re,time import queue import threading headers = ("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3107.4 Safari/537.36") operner = urllib.request.build_opener() operner.addheaders = [headers] urllib.request.install_opener(operner) urlque = queue.Queue() list_url = [] ###使用代理獲取網(wǎng)頁url內(nèi)容 def use_proxy(url): try: # proxy = urllib.request.ProxyHandler({'http':proxy_addr}) # operner = urllib.request.build_opener() # urllib.request.install_opener(operner) headers = ("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3107.4 Safari/537.36") operner = urllib.request.build_opener() operner.addheaders = [headers] urllib.request.install_opener(operner) data = urllib.request.urlopen(url).read().decode('utf-8') #print (data) return data except urllib.error.URLError as e: if hasattr(e,"code"): print (e.code) elif hasattr(e,"reason"): print (e.reason) except Exception as e: print ("exception"+str(e)) time.sleep(1) ###獲取文章的url連接,并將連接加入到隊(duì)列 class get_url(threading.Thread): def __init__(self,key,pagestart,pageend,urlque): threading.Thread.__init__(self) self.pagestart = pagestart self.pageend = pageend self.key = key self.urlque = urlque def run(self): try: keycode = urllib.parse.quote(self.key) for page in range(self.pagestart,self.pageend+1): url = "http://weixin.sogou.com/weixin?query=%s&_sug_type_=&s_from=input&_sug_=n&type=%d&page=1&ie=utf8" % (keycode,page) data = use_proxy(url) print ("data1的內(nèi)容是",data) listurl_pattern = '<h4>.*?("http://.*?)</h4>' result = re.compile(listurl_pattern,re.S).findall(data) print (result) if len(result) == 0: print ("沒有可用的url") sys.exit() for i in range(len(result)): res = result[i].replace("amp;","").split(" ")[0].replace("\"" ,"") #list_url.append(res) #加入列表 self.urlque.put(res) ##加入隊(duì)列 self.urlque.task_done() #return list_url except urllib.error.URLError as e: if hasattr(e, "code"): print(e.code) elif hasattr(e, "reason"): print(e.reason) except Exception as e: print ("exception:",e) ##根據(jù)url獲取文章內(nèi)容 class get_url_content(threading.Thread): def __init__(self,urlque): threading.Thread.__init__(self) self.urlque = urlque def run(self): fh2 = open("D:\\python-script\\1.html", 'wb') html1 = '''<!DOCTYPE html>\n<html xmlns="http://www.w3.org/1999/xhmtl">\n<head>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n<title>微信文章</title></head>\n<body>''' fh2.write(html1.encode("utf-8")) fh2.close() fh = open("D:\\python-script\\1.html", 'ab') while True: try: url = self.urlque.get() data_content = use_proxy(url) title_pattern = '<h3.*>.*?</h3>' result_title = re.compile(title_pattern, re.S).findall(data_content) ##標(biāo)題 res_title = result_title[0].replace("<h3 class=\"rich_media_title\" id=\"activity-name\">", "").replace("</h3>","").strip() content_pattern = 'id="js_content">(.*?)<div class="rich_media_tool" id="js_sg_bar">' content = re.compile(content_pattern, re.S).findall(data_content) #c = '<p >' # for i in content: # ##內(nèi)容 # c_content=i.replace(c, "").replace("<br /></p>", "").replace("</p>", "") fh.write(res_title.encode("utf-8")) for i in content: fh.write(i.strip().encode("utf-8")) except UnicodeEncodeError as e: continue fh.close() class contrl(threading.Thread): def __init__(self,urlqueue): threading.Thread.__init__(self) self.urlqueue = urlqueue while True: print ("程序正在執(zhí)行") if self.urlqueue.empty(): time.sleep(3) print ("程序執(zhí)行完畢") exit() if __name__ == '__main__': pagestart = 1 pageend = 2 key = "人工智能" get_url = get_url(key,pagestart,pageend,urlque) get_url.start() get_content = get_url_content(urlque) get_content.start() cntrol = contrl(urlque) cntrol.start()
以上是“Python中微信爬蟲的示例分析”這篇文章的所有內(nèi)容,感謝各位的閱讀!相信大家都有了一定的了解,希望分享的內(nèi)容對大家有所幫助,如果還想學(xué)習(xí)更多知識,歡迎關(guān)注億速云行業(yè)資訊頻道!
免責(zé)聲明:本站發(fā)布的內(nèi)容(圖片、視頻和文字)以原創(chuàng)、轉(zhuǎn)載和分享為主,文章觀點(diǎn)不代表本網(wǎng)站立場,如果涉及侵權(quán)請聯(lián)系站長郵箱:is@yisu.com進(jìn)行舉報,并提供相關(guān)證據(jù),一經(jīng)查實(shí),將立刻刪除涉嫌侵權(quán)內(nèi)容。