Java爬蟲實(shí)現(xiàn)爬取京東上的手機(jī)搜索頁面 HttpCliient+Jsoup

<dependencies>
 <dependency>
 <groupId>org.apache.httpcomponents</groupId>
 <artifactId>httpclient</artifactId>
 <version>4.5.3</version>
 </dependency>

 <dependency>
 <!-- jsoup HTML parser library @ https://jsoup.org/ -->
 <groupId>org.jsoup</groupId>
 <artifactId>jsoup</artifactId>
 <version>1.11.2</version>
 </dependency>

 <!-- https://mvnrepository.com/artifact/log4j/log4j -->
 <dependency>
 <groupId>log4j</groupId>
 <artifactId>log4j</artifactId>
 <version>1.2.17</version>
 </dependency>
 </dependencies>

log4j.rootLogger=INFO, Console 
 
#Console 
log4j.appender.Console=org.apache.log4j.ConsoleAppender
log4j.appender.Console.layout=org.apache.log4j.PatternLayout
log4j.appender.Console.layout.ConversionPattern=%d [%t] %-5p [%c] - %m%n

import java.io.IOException;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;

public class getHTML {
 //建立日志
 private static Logger log = Logger.getLogger(getHTML.class);
 private static String getH(String url) throws ClientProtocolException, IOException {
 //控制臺輸出日志，這樣每條訪問的URL都可以在控制臺上看到訪問情況
 log.info("正在解析" + url);
 
 /* 
  * 以下內(nèi)容為HttpClient建立連接的一般用法
  * 使用HttpClient建立客戶端
  * 使用get方法訪問指定URL
  * 獲得應(yīng)答
  * */
 
 CloseableHttpClient client = HttpClients.createDefault();
 HttpGet get = new HttpGet(url);
 CloseableHttpResponse response = client.execute(get);
 
 /*
  * 以下內(nèi)容為將HTML內(nèi)容轉(zhuǎn)化為String
  * 獲得應(yīng)答體
  * 將應(yīng)答體轉(zhuǎn)為String格式，此處使用了EntityUtils中的toString方法，編碼格式設(shè)置為"utf-8"
  * 完成后關(guān)閉客戶端與應(yīng)答
  * */
 HttpEntity entity = response.getEntity();
 String content;
 if (entity != null) {
  content = EntityUtils.toString(entity, "utf-8");
  client.close();
  response.close();
  return content;
 } else
  return null;
 }
 public static void urlControl(String baseurl, int page) throws ClientProtocolException, IOException {
 //設(shè)置當(dāng)前頁count
 int count = 1;
 //如果當(dāng)前頁小于想要爬取的頁數(shù)則執(zhí)行
 while (count < page) {
  //實(shí)際訪問的URL為不變的URL值拼接上URL變化的值
  String u = baseurl + (2 * count - 1) + "&click=0";
  //此處調(diào)用ParseHTML類中的方法對URL中的HTML頁面進(jìn)行處理，后面詳細(xì)介紹該類
  String content = ParseHTML.parse(getH(u)).toString();
  //此處調(diào)用WriteTo類中的方法對解析出來的內(nèi)容寫入到本地，后面詳細(xì)介紹該類
  WriteTo.writeto(content);
  count++;
 }
 }
}

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class ParseHTML {
 
 public static StringBuilder parse(String content)
 {
 //使用Jsoup中的parse方法對已經(jīng)轉(zhuǎn)換為String的HTML內(nèi)容進(jìn)行分析，返回值為Document類
 Document doc = Jsoup.parse(content);
 //使用選擇器select對需要找的元素進(jìn)行抓取，例如第一個select中選擇的是ul標(biāo)簽中class屬性等于gl-warp clearfix的內(nèi)容
 Elements ele = doc.select("ul[class = gl-warp clearfix]").select("li[class=gl-item]");
 //設(shè)置一個容器，用于裝各個屬性
 StringBuilder sb = new StringBuilder();
 //通過上一個選擇器可以獲得整個頁面中所有符合要求的元素，也即各款手機(jī)，下面則需要對每款手機(jī)進(jìn)行遍歷，獲取其屬性
 for (Element e : ele) {
  //此處對各個屬性的獲取參考了網(wǎng)上一篇爬取京東上內(nèi)容的文章，應(yīng)該有其他不同的寫法
  String id = e.attr("data-pid");
  String mingzi = e.select("div[class = p-name p-name-type-2]").select("a").select("em").text();
  String jiage = e.select("div[class=p-price]").select("strong").select("i").text();
  String pinglun = e.select("div[class=p-commit]").select("strong").select("a").text();
  //向容器中添加屬性
  sb.append(id+"\t");
  sb.append(mingzi+"\t");
  sb.append(jiage+"\t");
  sb.append(pinglun+"\t");
  sb.append("\r\n");
 }
 return sb;
 }
}

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;

public class WriteTo {
 // 設(shè)置文件存放的位置
 private static File f = new File("C:\\jingdong.txt");
 public static void writeto(String content) throws IOException {
 //使用續(xù)寫的方式，以免覆蓋前面寫入的內(nèi)容
 BufferedWriter bw = new BufferedWriter(new FileWriter(f, true));
 bw.append(content);
 bw.flush();
 bw.close();
 }
}

import java.io.IOException;
import org.apache.http.client.ClientProtocolException;

public class MainControl {
 public static void main(String[] args) throws ClientProtocolException, IOException {
 // TODO Auto-generated method stub
 String baseurl = "https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc="
  + "utf-8&qrst=1&rt=1&stop=1&vt=2&cid2=653&cid3=655&page=";
 int page = 5;//設(shè)置爬取頁數(shù)
 getHTML.urlControl(baseurl, page);
 }
}