溫馨提示×
您好，登錄后才能下訂單哦！
點(diǎn)擊 登錄注冊(cè) 即表示同意《億速云用戶(hù)服務(wù)條款》
您好，登錄后才能下訂單哦！
package main.java;
import java.io.*;
import java.util.LinkedList;
import java.util.List;
import java.util.zip.*;
import org.apache.commons.compress.archivers.ArchiveException;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import java.io.IOException;
import java.net.URI;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;;
/**
* 解壓tar.gz zip gz文件包 這里的數(shù)據(jù)源和輸出目錄都為HDFS
*
*/
public class GZipHdfs {
private BufferedOutputStream bufferedOutputStream;
String zipfileName = null;
public GZipHdfs(String fileName) {
this.zipfileName = fileName;
}
/*
* 執(zhí)行入口,rarFileName為需要解壓的文件路徑(具體到文件),destDir為解壓目標(biāo)路徑 路徑為HDFS
*/
public List<String> unTargzFile(String rarFileName, String destDir) throws IOException {
GZipHdfs GZipHdfs = new GZipHdfs(rarFileName);
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(destDir), conf);
boolean result = fs.isDirectory(new Path(destDir));
if (!result) {
fs.mkdirs(new Path(destDir));
}
String outputDirectory = destDir;
List<String> r = GZipHdfs.defUnTargzFile(outputDirectory, fs);
fs.close();
return r;
}
public List<String> defUnTargzFile(String outputDirectory, FileSystem fs) {
FileInputStream fis = null;
ArchiveInputStream in = null;
BufferedInputStream bufferedInputStream = null;
List<String> tarList = new LinkedList<String>();
try {
FSDataInputStream hdfsInputStream = fs.open(new Path(zipfileName));
GZIPInputStream is = new GZIPInputStream(new BufferedInputStream(
hdfsInputStream));
in = new ArchiveStreamFactory().createArchiveInputStream("tar", is);
bufferedInputStream = new BufferedInputStream(in);
TarArchiveEntry entry = (TarArchiveEntry) in.getNextEntry();
while (entry != null) {
String name = entry.getName();
String[] names = name.split("/");
String fileName = outputDirectory;
for (int i = 0; i < names.length; i++) {
String str = names[i];
fileName = fileName + "/" + str;
}
FSDataOutputStream hdfsOutStream = fs.create(new Path(fileName));
bufferedOutputStream = new BufferedOutputStream(
hdfsOutStream);
int b;
while ((b = bufferedInputStream.read()) != -1) {
bufferedOutputStream.write(b);
}
bufferedOutputStream.flush();
bufferedOutputStream.close();
entry = (TarArchiveEntry) in.getNextEntry();
tarList.add(name);
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (ArchiveException e) {
e.printStackTrace();
} finally {
try {
if (bufferedInputStream != null) {
bufferedInputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return tarList;
}
/*
* 執(zhí)行入口,rarFileName為需要解壓的文件路徑(具體到文件),destDir為解壓目標(biāo)路徑 路徑為HDFS
*/
public List<String> unZipFile(String rarFileName, String destDir) throws IOException {
GZipHdfs GZipHdfs = new GZipHdfs(rarFileName);
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(destDir), conf);
boolean result = fs.isDirectory(new Path(destDir));
if (!result) {
fs.mkdirs(new Path(destDir));
}
String outputDirectory = destDir;
List<String> r = GZipHdfs.defUnZipFile(outputDirectory, fs);
fs.close();
return r;
}
public List<String> defUnZipFile(String outputDirectory, FileSystem fs) {
FileInputStream fis = null;
ArchiveInputStream in = null;
BufferedInputStream bufferedInputStream = null;
List<String> zipList = new LinkedList<String>();
try {
FSDataInputStream hdfsInputStream = fs.open(new Path(zipfileName));
ZipInputStream is = new ZipInputStream(new BufferedInputStream(
hdfsInputStream));
bufferedInputStream = new BufferedInputStream(is);
ZipEntry entry =is.getNextEntry();
while (entry != null) {
String name = entry.getName();
String[] names = name.split("/");
String fileName = outputDirectory;
for (int i = 0; i < names.length; i++) {
String str = names[i];
fileName = fileName + "/" + str;
}
FSDataOutputStream hdfsOutStream = fs.create(new Path(fileName));
bufferedOutputStream = new BufferedOutputStream(
hdfsOutStream);
int b;
while ((b = bufferedInputStream.read()) != -1) {
bufferedOutputStream.write(b);
}
bufferedOutputStream.flush();
bufferedOutputStream.close();
entry = (ZipEntry) is.getNextEntry();
zipList.add(name);
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (bufferedInputStream != null) {
bufferedInputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return zipList;
}
/*
* 執(zhí)行入口,rarFileName為需要解壓的文件路徑(具體到文件),destDir為解壓目標(biāo)路徑 路徑為HDFS
*/
public List<String> unGZipFile(String rarFileName, String destDir) throws IOException {
GZipHdfs GZipHdfs = new GZipHdfs(rarFileName);
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(destDir), conf);
boolean result = fs.isDirectory(new Path(destDir));
if (!result) {
fs.mkdirs(new Path(destDir));
}
String outputDirectory = destDir;
List<String> r = GZipHdfs.defUnGZipFile(outputDirectory, fs);
fs.close();
return r;
}
public List<String> defUnGZipFile(String outputDirectory, FileSystem fs) {
FileInputStream fis = null;
ArchiveInputStream in = null;
BufferedInputStream bufferedInputStream = null;
List<String> tarList = new LinkedList<String>();
try {
FSDataInputStream hdfsInputStream = fs.open(new Path(zipfileName));
GzipCompressorInputStream is = new GzipCompressorInputStream(new BufferedInputStream(
hdfsInputStream));
bufferedInputStream = new BufferedInputStream(is);
String[] nameList = zipfileName.split("/");
String name=nameList[nameList.length-1].replace(".gz","");
String fileName = outputDirectory+"/"+name;
FSDataOutputStream hdfsOutStream = fs.create(new Path(fileName));
bufferedOutputStream = new BufferedOutputStream(
hdfsOutStream);
int b;
while ((b = bufferedInputStream.read()) != -1) {
bufferedOutputStream.write(b);
}
bufferedOutputStream.flush();
bufferedOutputStream.close();
tarList.add(name);
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (bufferedInputStream != null) {
bufferedInputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return tarList;
}
}
免責(zé)聲明：本站發(fā)布的內(nèi)容（圖片、視頻和文字）以原創(chuàng)、轉(zhuǎn)載和分享為主，文章觀點(diǎn)不代表本網(wǎng)站立場(chǎng)，如果涉及侵權(quán)請(qǐng)聯(lián)系站長(zhǎng)郵箱：is@yisu.com進(jìn)行舉報(bào)，并提供相關(guān)證據(jù)，一經(jīng)查實(shí)，將立刻刪除涉嫌侵權(quán)內(nèi)容。
億速云公眾號(hào)
手機(jī)網(wǎng)站二維碼
