Eclipse集成hadoop插件開發(fā)環(huán)境

首先在win10下搭建好hadoop的環(huán)境，hadoop可以運行
解壓hadoop2.7.7的安裝包和源碼包，解壓后創(chuàng)建一個空的目錄，把解壓的源碼包，安裝包下share/hadoop下的除了kms的目錄包外其他包下的所有jar包都拷貝到剛新建的空目錄中。大概有120多個
把之前安裝win10的hadoop/bin下的hadoop.dll 放到c:windows/system32下,重啟電腦
檢查之前安裝的本地hadoop環(huán)境是否配置了hadoop的環(huán)境變量和hadoop的HADOOP_USER_NAME默認用root。把hadoop.dll文件放到C盤windows/system32下
在安裝eclipse路徑下plugins,dropins，把hadoop-eclipse-plugin-2.6.0.jar(可以對應(yīng)下載自己版本的插件)放到該路徑/eclipse/plugins/ 和/eclipse/dropins下，啟動eclipse

安裝成功

?6.在ecplise里面window->preferences里找到 Hadoop Map/Reduce 把本地安裝的hadoop路徑指定到這。
?
7.先確認hadoop集群是否啟動，然后

在第二步的Map/Reduce Locations里新建


然后點完成，就能看到ecplise連接hadoop了

如果看不到點擊localhadoop右鍵reconnection 重新連
8.導(dǎo)入包
eclipse里面 windows-》preferences->Java ->bulid path-》user libraries

然后 ecplise 創(chuàng)建一個工程: File -> new -> project - >java -> java project


打jar包的時候不用把hadoopLib jar打進去只用打程序
點到工程，把JUnit4包引進去。然后創(chuàng)建一個conf文件夾在工程里，在conf目錄下再創(chuàng)建一個HA目錄

把hadoop集群中的core-site.xml 和 hdfs-site.xml 添加到HA目錄中
點中HA文件夾

測試代碼:
package com.test.hadoop.hdfs;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

public class TestHDFS {

Configuration conf;
FileSystem fs;

@Before
public void conn() throws IOException {
    conf = new Configuration(true);
    fs = FileSystem.get(conf);
}

@After
public void close() throws IOException {
    fs.close();
}

//創(chuàng)建目錄
@Test
public void mkDir() throws IOException {
    Path ifile = new Path("/ecpliseMkdir");
    if(fs.exists(ifile)) {
        fs.delete(ifile, true);
    }
    fs.mkdirs(ifile);
}

//上傳文件
@Test
public void upload() throws IOException {
    Path ifile = new Path("/ecpliseMkdir/hello.txt");
    FSDataOutputStream output = fs.create(ifile);
    InputStream input = new BufferedInputStream(new FileInputStream(new File("d:\\ywcj_chnl_risk_map_estimate_model.sql")));
    IOUtils.copyBytes(input, output, conf, true);
}

//下載
@Test
public void downLocal() throws IOException {
    Path ifile = new Path("/ecpliseMkdir/hello.txt");
    FSDataInputStream open = fs.open(ifile);
    File newFile = new File("d:\\test.txt");
    if(!newFile.exists()) {
        newFile.createNewFile();
    }
    BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream(newFile));
    IOUtils.copyBytes(open, output, conf, true);

}

//獲取block 塊信息
@Test
public void blockInfo() throws IOException {
    Path ifile = new Path("/ecpliseMkdir/hello.txt");
    FileStatus fsu = fs.getFileStatus(ifile);
    BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(ifile, 0, fsu.getLen());
    for(BlockLocation b : fileBlockLocations) {
        System.out.println(b);
    }
}

//刪除文件
@Test
public void deleteFile() throws IOException {
    Path ifile = new Path("/ecpliseMkdir/hello.txt");
    boolean delete = fs.delete(ifile, true);
    if(delete) {
        System.out.println("刪除成功---------");
    }
}

}