您好,登錄后才能下訂單哦!
這篇文章給大家分享的是有關(guān)python爬蟲項目的示例分析的內(nèi)容。小編覺得挺實用的,因此分享給大家做個參考,一起跟隨小編過來看看吧。
爬取內(nèi)涵段子:
#encoding=utf-8 import urllib2 import re class neihanba(): def spider(self): ''' 爬蟲的主調(diào)度器 ''' isflow=True#判斷是否進(jìn)行下一頁 page=1 while isflow: url="http://www.neihanpa.com/article/list_5_"+str(page)+".html" html=self.load(url) self.deal(html,page) panduan=raw_input("是否繼續(xù)(y/n)!") if panduan=="y": isflow=True page+=1 else: isflow=False def load(self,url): ''' 針對url地址進(jìn)行全部爬去 :param url: url地址 :return: 返回爬去的內(nèi)容 ''' header = { "User-Agent": " Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36" } request = urllib2.Request(url, headers=header) response = urllib2.urlopen(request) html = response.read() return html def deal(self,html,page): ''' 對之前爬去的內(nèi)容進(jìn)行正則匹配,匹配出標(biāo)題和正文內(nèi)容 :param html:之前爬去的內(nèi)容 :param page: 正在爬去的頁碼 ''' parrten=re.compile('<li class="piclist\d+">(.*?)</li>',re.S) titleList=parrten.findall(html) for title in titleList: parrten1=re.compile('<a href="/article/\d+.html" rel="external nofollow" >(.*)</a>') ti1=parrten1.findall(title) parrten2=re.compile('<div class="f18 mb20">(.*?)</div>',re.S) til2=parrten2.findall(title) for t in ti1: tr=t.replace("<b>","").replace("</b>","") self.writeData(tr,page) for t in til2: tr=t.replace("<p>","").replace("</p>","").replace("<br>","").replace("<br />","").replace("&ldquo","\"").replace("&rdquo","\"") self.writeData(tr,page) def writeData(self,context,page): ''' 將最終爬去的內(nèi)容寫入文件中 :param context: 匹配好的內(nèi)容 :param page: 當(dāng)前爬去的頁碼數(shù) ''' fileName = "di" + str(page) + "yehtml.txt" with open(fileName, "a") as file: file.writelines(context + "\n") if __name__ == '__main__': n=neihanba() n.spider()
爬取智聯(lián):
#encoding=utf-8 import urllib import urllib2 import re class zhiLian(): def spider(self,position,workPlace): ''' 爬蟲的主調(diào)度器 :param position: 職位 :param workPlace: 工作地點 ''' url="http://sou.zhaopin.com/jobs/searchresult.ashx?" url+=urllib.urlencode({"jl":workPlace}) url+="&" url+=urllib.urlencode({"kw":position}) isflow=True#是否進(jìn)行下一頁的爬去 page=1 while isflow: url+="&"+str(page) html=self.load(url) self.deal1(html,page) panduan = raw_input("是否繼續(xù)爬蟲下一頁(y/n)!") if panduan == "y": isflow = True page += 1 else: isflow = False def load(self,url): ''' 針對url地址進(jìn)行全部爬去 :param url: url地址 :return: 返回爬去的內(nèi)容 ''' header = { "User-Agent": " Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36" } request = urllib2.Request(url, headers=header) response = urllib2.urlopen(request) html = response.read() return html def deal1(self,html,page): ''' 對之前爬去的內(nèi)容進(jìn)行正則匹配,匹配職位所對應(yīng)的鏈接 :param html:之前爬去的內(nèi)容 :param page: 正在爬去的頁碼 ''' parrten=re.compile('<a\s+\s+par="ssidkey=y&ss=\d+&ff=\d+&sg=\w+&so=\d+"\s+href="(.*?)" rel="external nofollow" target="_blank">.*?</a>',re.S) til=parrten.findall(html)#爬去鏈接 for t in til: self.deal2(t,page) def deal2(self,t,page): ''' 進(jìn)行二次爬蟲,然后在新的頁面中對公司、薪資、工作經(jīng)驗進(jìn)行匹配 :param t: url地址 :param page: 當(dāng)前匹配的頁數(shù) ''' html=self.load(t)#返回二次爬蟲的內(nèi)容 parrten1=re.compile('<a\s+onclick=".*?"\s+href=".*?" rel="external nofollow" \s+target="_blank">(.*?)\s+.*?<img\s+class=".*?"\s+src=".*?"\s+border="\d+"\s+vinfo=".*?"></a>',re.S) parrten2=re.compile('<li><span>職位月薪:</span><strong>(.*?) <a.*?>.*?</a></strong></li>',re.S) parrent3=re.compile('<li><span>工作經(jīng)驗:</span><strong>(.*?)</strong></li>',re.S) til1=parrten1.findall(html) til2=parrten2.findall(html) til3=parrent3.findall(html) str="" for t in til1: t=t.replace('<img title="專屬頁面" src="//img03.zhaopin.cn/2012/img/jobs/icon.png" border="0" />',"") str+=t str+="\t" for t in til2: str+=t str += "\t" for t in til3: str+=t self.writeData(str,page) def writeData(self,context,page): ''' 將最終爬去的內(nèi)容寫入文件中 :param context: 匹配好的內(nèi)容 :param page: 當(dāng)前爬去的頁碼數(shù) ''' fileName = "di" + str(page) + "yehtml.txt" with open(fileName, "a") as file: file.writelines(context + "\n") if __name__ == '__main__': position=raw_input("請輸入職位:") workPlace=raw_input("請輸入工作地點:") z=zhiLian() z.spider(position,workPlace)
爬取貼吧:
#encoding=utf-8 import urllib import urllib2 import re class teiba(): def spider(self,name,startPage,endPage): url="http://tieba.baidu.com/f?ie=utf-8&" url+=urllib.urlencode({"kw":name}) for page in range(startPage,endPage+1): pn=50*(page-1) urlFull=url+"&"+urllib.urlencode({"pn":pn}) html=self.loadPage(url) self.dealPage(html,page) def loadPage(self,url): header={ "User-Agent":" Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36" } request=urllib2.Request(url,headers=header) response=urllib2.urlopen(request) html=response.read() return html def dealPage(self,html,page): partten=re.compile(r'<a\s+rel="noreferrer"\s+href="/p/\d+" rel="external nofollow" \s+title=".*?"\s+target="_blank" class="j_th_tit\s+">(.*?)</a>',re.S) titleList=partten.findall(html) rstr=r'<span\s+class="topic-tag"\s+data-name=".*?">#(.*?)#</span>' for title in titleList: title=re.sub(rstr,"",title) self.writePage(title,page) def writePage(self,context,page): fileName="di"+str(page)+"yehtml.txt" with open(fileName,"a") as file: file.writelines(context+"\n") if __name__ == '__main__': name=raw_input("請輸入貼吧名:") startPage=raw_input("請輸入起始頁:") endPage=raw_input("請輸入終止頁:") t=teiba() t.spider(name,int(startPage),int(endPage))
感謝各位的閱讀!關(guān)于“python爬蟲項目的示例分析”這篇文章就分享到這里了,希望以上內(nèi)容可以對大家有一定的幫助,讓大家可以學(xué)到更多知識,如果覺得文章不錯,可以把它分享出去讓更多的人看到吧!
免責(zé)聲明:本站發(fā)布的內(nèi)容(圖片、視頻和文字)以原創(chuàng)、轉(zhuǎn)載和分享為主,文章觀點不代表本網(wǎng)站立場,如果涉及侵權(quán)請聯(lián)系站長郵箱:is@yisu.com進(jìn)行舉報,并提供相關(guān)證據(jù),一經(jīng)查實,將立刻刪除涉嫌侵權(quán)內(nèi)容。