幾行Python代碼爬取3000+上市公司的信息

import pandas as pdimport csvfor i in range(1,178): # 爬取全部頁  tb = pd.read_html('http://s.askci.com/stock/a/?reportTime=2017-12-31&pageNum=%s' % (str(i)))[3]   tb.to_csv(r'1.csv', mode='a', encoding='utf_8_sig', header=1, index=0)

import requestsimport pandas as pdfrom bs4 import BeautifulSoupfrom lxml import etreeimport timeimport pymysqlfrom sqlalchemy import create_enginefrom urllib.parse import urlencode # 編碼 URL 字符串start_time = time.time() #計(jì)算程序運(yùn)行時(shí)間def get_one_page(i):  try:    headers = {      'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'    }    paras = {    'reportTime': '2017-12-31',    #可以改報(bào)告日期，比如2018-6-30獲得的就是該季度的信息    'pageNum': i  #頁碼    }    url = 'http://s.askci.com/stock/a/?' + urlencode(paras)    response = requests.get(url,headers = headers)    if response.status_code == 200:      return response.text    return None  except RequestException:    print('爬取失敗')def parse_one_page(html):  soup = BeautifulSoup(html,'lxml')  content = soup.select('#myTable04')[0] #[0]將返回的list改為bs4類型  tbl = pd.read_html(content.prettify(),header = 0)[0]  # prettify()優(yōu)化代碼,[0]從pd.read_html返回的list中提取出DataFrame  tbl.rename(columns = {'序號(hào)':'serial_number', '股票代碼':'stock_code', '股票簡稱':'stock_abbre', '公司名稱':'company_name', '省份':'province', '城市':'city', '主營業(yè)務(wù)收入(201712)':'main_bussiness_income', '凈利潤(201712)':'net_profit', '員工人數(shù)':'employees', '上市日期':'listing_date', '招股書':'zhaogushu', '公司財(cái)報(bào)':'financial_report', '行業(yè)分類':'industry_classification', '產(chǎn)品類型':'industry_type', '主營業(yè)務(wù)':'main_business'},inplace = True)  return tbldef generate_mysql():  conn = pymysql.connect(    host='localhost',    user='root',    password='******',    port=3306,    charset = 'utf8',     db = 'wade')  cursor = conn.cursor()  sql = 'CREATE TABLE IF NOT EXISTS listed_company (serial_number INT(20) NOT NULL,stock_code INT(20) ,stock_abbre VARCHAR(20) ,company_name VARCHAR(20) ,province VARCHAR(20) ,city VARCHAR(20) ,main_bussiness_income VARCHAR(20) ,net_profit VARCHAR(20) ,employees INT(20) ,listing_date DATETIME(0) ,zhaogushu VARCHAR(20) ,financial_report VARCHAR(20) , industry_classification VARCHAR(20) ,industry_type VARCHAR(100) ,main_business VARCHAR(200) ,PRIMARY KEY (serial_number))'  cursor.execute(sql)  conn.close()def write_to_sql(tbl, db = 'wade'):  engine = create_engine('mysql+pymysql://root:******@localhost:3306/{0}?charset=utf8'.format(db))  try:    tbl.to_sql('listed_company2',con = engine,if_exists='append',index=False)    # append表示在原有表基礎(chǔ)上增加，但該表要有表頭  except Exception as e:    print(e)def main(page):  generate_mysql()  for i in range(1,page):     html = get_one_page(i)    tbl = parse_one_page(html)    write_to_sql(tbl)# # 單進(jìn)程if __name__ == '__main__':    main(178)  endtime = time.time()-start_time  print('程序運(yùn)行了%.2f秒' %endtime)# 多進(jìn)程from multiprocessing import Poolif __name__ == '__main__':   pool = Pool(4)   pool.map(main, [i for i in range(1,178)]) #共有178頁  endtime = time.time()-start_time  print('程序運(yùn)行了%.2f秒' %(time.time()-start_time))