Java爬取網(wǎng)站源代碼和鏈接代碼實(shí)例

package com.test;

import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
 
/**
 * java實(shí)現(xiàn)爬蟲(chóng)
 */
public class SpiderDemo1 {
 
  //網(wǎng)站主網(wǎng)頁(yè)鏈接
  private final static String theURL = "http://www.jyyishu.cn";
  //今日日期，用于標(biāo)記日志
  private final static String theTIME = new SimpleDateFormat("yyyy-MM-dd").format(new Date());
  //網(wǎng)頁(yè)鏈接文件路徑
  private final static String theFILE = "L:/html/jy1/" + theTIME + "/URL.txt";
  //網(wǎng)頁(yè)源碼路徑
  private final static String thePATH = "L:/html/jy1/" + theTIME + "/code";
  //正則表達(dá)式，用于判斷是否是一個(gè)網(wǎng)址
  private final static String theREGEX= "(http|https)://[\\w+\\.?/?]+\\.[A-Za-z]+";
 
  /**
   * 啟動(dòng)類(lèi)
   * @param args
   */
  public static void main(String[] args) {
    found();
    System.out.println("網(wǎng)站爬取完成");
  }
 
 
  public static void found() {
    PrintWriter pw = null;
    try{
      //創(chuàng)建文件目錄
      File fileDir = new File(thePATH);
      if (!fileDir.exists()) {
        fileDir.mkdirs();
      }
 
      //創(chuàng)建網(wǎng)站網(wǎng)頁(yè)鏈接文件
      pw = new PrintWriter(new FileWriter(theFILE),true);
 
      //使用遞歸遍歷網(wǎng)站的每個(gè)網(wǎng)頁(yè)
      spiderURL(theURL, pw);
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      try {
        if(pw != null) {
          pw.close();
        }
      } catch(Exception e) {
        e.printStackTrace();
      }
    }
  }
 
 
  /**
   * 爬取該網(wǎng)頁(yè)源碼和網(wǎng)頁(yè)內(nèi)連接
   * @param url 該網(wǎng)頁(yè)網(wǎng)址
   * @param tpw 對(duì)網(wǎng)站網(wǎng)頁(yè)網(wǎng)址文件連接的io流
   */
  public static void spiderURL(String url, PrintWriter tpw){
    URL realURL=null;
    URLConnection connection=null;
    BufferedReader br=null;
    PrintWriter pw=null;
    PrintWriter pw1=null;
 
    Pattern pattern=Pattern.compile(theREGEX);
    try{
      realURL=new URL(url);
      connection=realURL.openConnection();
 
      //生成文件夾
      String src = thePATH + url.substring(theURL.length());
      File fileDir = new File(src);
      if (!fileDir.exists()) {
        fileDir.mkdirs();
      }
 
      //生成源代碼文件
      pw = new PrintWriter(new FileWriter(src + "/Test.txt"),true);
      pw1 = tpw;
 
      //爬取網(wǎng)頁(yè)文件
      br=new BufferedReader(new InputStreamReader(connection.getInputStream()));
      String line=null;
      while((line=br.readLine())!=null){
        //把爬取的源碼寫(xiě)入文件
        pw.println(line);
        System.out.println("爬取網(wǎng)頁(yè)" + url + "成功");
        Matcher matcher=pattern.matcher(line);
        //判斷是否是一個(gè)網(wǎng)址
        while(matcher.find()){
          //判斷網(wǎng)址是否以網(wǎng)站主網(wǎng)址為前綴，防止爬到其他網(wǎng)站,并判斷是否和原先網(wǎng)址重復(fù)
          if(matcher.group().startsWith(theURL) && examine(matcher.group())) {
            //把爬取的網(wǎng)址寫(xiě)入文件
            pw1.println(matcher.group());
            spiderURL(matcher.group(), pw1);
          }
        }
        System.out.println("網(wǎng)頁(yè)" + url + "內(nèi)鏈接爬取完成");
      }
  }catch(Exception e){
    e.printStackTrace();
  }finally {
      try {
        if(br != null) {
          br.close();
        }
        if(pw != null) {
          pw.close();
        }
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }
 
 
  /**
   * 判斷是否和以儲(chǔ)存網(wǎng)址相同
   * @param str 需判斷的網(wǎng)址
   * @return 是否重復(fù)
   */
  public static boolean examine(String str) {
    BufferedReader br = null;
    String str1;
    try {
      br = new BufferedReader(new FileReader(theFILE));
 
//      //針對(duì)該網(wǎng)站無(wú)用網(wǎng)頁(yè)的屏蔽
//      if(str.startsWith("http://www.jyyishu.cn/artnews/")) {
//        return false;
//      }
      
      //循環(huán)文件中每一行的網(wǎng)址，判斷是否重復(fù)，重復(fù)則退出
      while((str1 = br.readLine()) != null) {
        if(str.equals(str1)) {
          return false;
        }
      }
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      try{
        if(br != null) {
          br.close();
        }
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
    return true;
  }
}