Python爬取網(wǎng)頁信息的示例

def get_url1():
 urls=[]
 # A,'B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z'
 a=['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z']　　　　#自動生成A~Z的鏈接
 for i in a:
  urls.append("https://nameberry.com/search/baby_names_starting_with/%s" %i)
 dp=pd.DataFrame(urls)
 dp.to_csv("A~Z_Link1.csv",mode="a",encoding='utf_8_sig')　　　　#循環(huán)用于在每個字母鏈接下，調(diào)用爬取名字鏈接的頁面的函數(shù)，即函數(shù)嵌套
 for j in urls:
  get_pages_Html(j)
 return urls

#獲取頁數(shù)
def get_pages_Html(url1):
 req = requests.get(url1)
 soup=BeautifulSoup(req.text)
#異常處理，為解決頁面不存在多頁的問題，使用re正則表達(dá)式獲取頁面數(shù)
 try:
  lastpage = soup.find(class_="last").find("a")['href']
  str1='{}'.format(lastpage)
  b=re.findall('\\d+', str1 )
  for page in b:
   num=page
 except:
  num=1
 get_pages(num,url1)
 return num

def get_pages(n,url):
 pages=[]
 for k in range(1,int(n)+1):
  pages.append("{}?page={}".format(url,k))
 dp=pd.DataFrame(pages)
 dp.to_csv("NUM_pages_1.csv",mode="a",encoding='utf_8_sig')
  #函數(shù)調(diào)用
 for l in pages:
  parse_HTML2(l)
 return pages


# 名字的鏈接，根據(jù)網(wǎng)頁源碼的標(biāo)簽，確定名字鏈接的位置
def parse_HTML2(url2):
 try:
  req = requests.get(url2)
  req.encoding = req.apparent_encoding
  soup = BeautifulSoup(req.text)
 except:
  dp=pd.DataFrame(url2)
  dp.to_csv("Error_pages_1.csv",mode="a",encoding='utf_8_sig')
 name_data_l=[]
 error=[]
 li_list = soup.find_all('li',class_="Listing-name pt-15 pb-15 bdb-gray-light w-100pct flex border-highlight")
 try:
  for li in li_list:
   nameList=li.find('a',class_='flex-1')['href']
   name_data_l.append('https://nameberry.com/'+nameList)
   time.sleep(1)
  cun(name_data_l,'Name_List_1')
 except:
  dp=pd.DataFrame(name_data_l)
  dp.to_csv("Error_Name_List_1.csv",mode="a",encoding='utf_8_sig')
  # cun(url2,'Error_link_Q')
 # dp=pd.DataFrame(name_data_l)
 # dp.to_csv("Name_List.csv",mode="a",encoding='utf_8_sig')
 # for i in name_data_l:
 #  parse_HTML3(i)
 return name_data_l

# 名字里的內(nèi)容
def parse_HTML3(url3):
 count=0
 req = requests.get(url3)
 req.encoding = req.apparent_encoding
 soup = BeautifulSoup(req.text)
 error=[]
 try:
  Name=soup.find('h2',class_='first-header').find("a").get_text().replace(",","").replace("\n","")
 except:
  error.append(url3)
  cun(error,"Error_Link_Comment")
 li_list = soup.find_all('div',class_="comment")
 for li in li_list:
  Title=li.find("h5").get_text().replace(",","").replace("\n","")
  Time=li.find("p",class_='meta').get_text().replace(",","").replace("\n","")
  Comments=li.find("div",class_='comment-text').get_text().replace(",","").replace("\n","")
  dic2={
   "Name":Name,
   "Title":Title,
   "Time":Time,
   "Comments":Comments
  }
  time.sleep(1)
  count=count+1
  save_to_csv(dic2,"Name_data_comment")
  print(count)
 return 1