溫馨提示×
您好，登錄后才能下訂單哦！
點(diǎn)擊 登錄注冊 即表示同意《億速云用戶服務(wù)條款》
您好，登錄后才能下訂單哦！
這篇文章主要介紹Hadoop編程基于MR程序如何實(shí)現(xiàn)倒排索引，文中介紹的非常詳細(xì)，具有一定的參考價(jià)值，感興趣的小伙伴們一定要看完！
一、數(shù)據(jù)準(zhǔn)備
1、輸入文件數(shù)據(jù)
這里我們準(zhǔn)備三個(gè)輸入文件，分別如下所示
a.txt
hello tom hello jerry hello tom
b.txt
hello jerry hello jerry tom jerry
c.txt
hello jerry hello tom
2、最終輸出文件數(shù)據(jù)
最終輸出文件的結(jié)果為：
[plain] view plain copy hello c.txt-->2 b.txt-->2 a.txt-->3 jerry c.txt-->1 b.txt-->3 a.txt-->1 tom c.txt-->1 b.txt-->1 a.txt-->2
二、倒排索引過程分析
根據(jù)輸入文件數(shù)據(jù)和最終的輸出文件結(jié)果可知，此程序需要利用兩個(gè)MR實(shí)現(xiàn)，具體流程可總結(jié)歸納如下：
-------------第一步Mapper的輸出結(jié)果格式如下：--------------------
context.wirte("hello->a.txt", "1")
context.wirte("hello->a.txt", "1")
context.wirte("hello->a.txt", "1")
context.wirte("hello->b.txt", "1")
context.wirte("hello->b.txt", "1")
context.wirte("hello->c.txt", "1")
context.wirte("hello->c.txt", "1")
-------------第一步Reducer的得到的輸入數(shù)據(jù)格式如下：-------------
<"hello->a.txt", {1,1,1}>
<"hello->b.txt", {1,1}>
<"hello->c.txt", {1,1}>
-------------第一步Reducer的輸出數(shù)據(jù)格式如下---------------------
context.write("hello->a.txt", "3")
context.write("hello->b.txt", "2")
context.write("hello->c.txt", "2")
-------------第二步Mapper得到的輸入數(shù)據(jù)格式如下：-----------------
context.write("hello->a.txt", "3")
context.write("hello->b.txt", "2")
context.write("hello->c.txt", "2")
-------------第二步Mapper輸出的數(shù)據(jù)格式如下：--------------------
context.write("hello", "a.txt->3")
context.write("hello", "b.txt->2")
context.write("hello", "c.txt->2")
-------------第二步Reducer得到的輸入數(shù)據(jù)格式如下：-----------------
<"hello", {"a.txt->3", "b.txt->2", "c.txt->2"}>
-------------第二步Reducer輸出的數(shù)據(jù)格式如下：-----------------
context.write("hello", "a.txt->3 b.txt->2 c.txt->2")
最終結(jié)果為：
hello a.txt->3 b.txt->2 c.txt->2三、程序開發(fā)
3.1、第一步MR程序與輸入輸出
package com.lyz.hdfs.mr.ii;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
* 倒排索引第一步Map Reduce程序,此處程序?qū)⑺械腗ap/Reduce/Runner程序放在一個(gè)類中
* @author liuyazhuang
*
*/
public class InverseIndexStepOne {
/**
* 完成倒排索引第一步的mapper程序
* @author liuyazhuang
*
*/
public static class StepOneMapper extends Mapper<LongWritable, Text, Text, LongWritable>{
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, LongWritable>.Context context)
throws IOException, InterruptedException {
//獲取一行數(shù)據(jù)
String line = value.toString();
//切分出每個(gè)單詞
String[] fields = StringUtils.split(line, " ");
//獲取數(shù)據(jù)的切片信息
FileSplit fileSplit = (FileSplit) context.getInputSplit();
//根據(jù)切片信息獲取文件名稱
String fileName = fileSplit.getPath().getName();
for(String field : fields){
context.write(new Text(field + "-->" + fileName), new LongWritable(1));
}
}
}
/**
* 完成倒排索引第一步的Reducer程序
* 最終輸出結(jié)果為：
* hello-->a.txt 3
hello-->b.txt 2
hello-->c.txt 2
jerry-->a.txt 1
jerry-->b.txt 3
jerry-->c.txt 1
tom-->a.txt 2
tom-->b.txt 1
tom-->c.txt 1
* @author liuyazhuang
*
*/
public static class StepOneReducer extends Reducer<Text, LongWritable, Text, LongWritable>{
@Override
protected void reduce(Text key, Iterable<LongWritable> values,
Reducer<Text, LongWritable, Text, LongWritable>.Context context) throws IOException, InterruptedException {
long counter = 0;
for(LongWritable value : values){
counter += value.get();
}
context.write(key, new LongWritable(counter));
}
}
//運(yùn)行第一步的MR程序
public static void main(String[] args) throws Exception{
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(InverseIndexStepOne.class);
job.setMapperClass(StepOneMapper.class);
job.setReducerClass(StepOneReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
FileInputFormat.addInputPath(job, new Path("D:/hadoop_data/ii"));
FileOutputFormat.setOutputPath(job, new Path("D:/hadoop_data/ii/result"));
job.waitForCompletion(true);
}
}3.1.1 輸入數(shù)據(jù)
a.txt
hello tom hello jerry hello tom
b.txt
hello jerry hello jerry tom jerry
c.txt
hello jerry hello tom
3.1.2
輸出結(jié)果：
hello-->a.txt 3 hello-->b.txt 2 hello-->c.txt 2 jerry-->a.txt 1 jerry-->b.txt 3 jerry-->c.txt 1 tom-->a.txt 2 tom-->b.txt 1 tom-->c.txt 1
3.2 第二步MR程序與輸入輸出
package com.lyz.hdfs.mr.ii;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
* 倒排索引第二步Map Reduce程序,此處程序?qū)⑺械腗ap/Reduce/Runner程序放在一個(gè)類中
* @author liuyazhuang
*
*/
public class InverseIndexStepTwo {
/**
* 完成倒排索引第二步的mapper程序
*
* 從第一步MR程序中得到的輸入信息為：
* hello-->a.txt 3
hello-->b.txt 2
hello-->c.txt 2
jerry-->a.txt 1
jerry-->b.txt 3
jerry-->c.txt 1
tom-->a.txt 2
tom-->b.txt 1
tom-->c.txt 1
* @author liuyazhuang
*
*/
public static class StepTwoMapper extends Mapper<LongWritable, Text, Text, Text>{
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] fields = StringUtils.split(line, "\t");
String[] wordAndFileName = StringUtils.split(fields[0], "-->");
String word = wordAndFileName[0];
String fileName = wordAndFileName[1];
long counter = Long.parseLong(fields[1]);
context.write(new Text(word), new Text(fileName + "-->" + counter));
}
}
/**
* 完成倒排索引第二步的Reducer程序
* 得到的輸入信息格式為:
* <"hello", {"a.txt->3", "b.txt->2", "c.txt->2"}>,
* 最終輸出結(jié)果如下：
* hello c.txt-->2 b.txt-->2 a.txt-->3
jerry c.txt-->1 b.txt-->3 a.txt-->1
tom c.txt-->1 b.txt-->1 a.txt-->2
* @author liuyazhuang
*
*/
public static class StepTwoReducer extends Reducer<Text, Text, Text, Text>{
@Override
protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
String result = "";
for(Text value : values){
result += value + " ";
}
context.write(key, new Text(result));
}
}
//運(yùn)行第一步的MR程序
public static void main(String[] args) throws Exception{
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(InverseIndexStepTwo.class);
job.setMapperClass(StepTwoMapper.class);
job.setReducerClass(StepTwoReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path("D:/hadoop_data/ii/result/part-r-00000"));
FileOutputFormat.setOutputPath(job, new Path("D:/hadoop_data/ii/result/final"));
job.waitForCompletion(true);
}
}3.2.1 輸入數(shù)據(jù)
hello-->a.txt 3 hello-->b.txt 2 hello-->c.txt 2 jerry-->a.txt 1 jerry-->b.txt 3 jerry-->c.txt 1 tom-->a.txt 2 tom-->b.txt 1 tom-->c.txt 1
3.2.2 輸出結(jié)果
hello c.txt-->2 b.txt-->2 a.txt-->3 jerry c.txt-->1 b.txt-->3 a.txt-->1 tom c.txt-->1 b.txt-->1 a.txt-->2
以上是“Hadoop編程基于MR程序如何實(shí)現(xiàn)倒排索引”這篇文章的所有內(nèi)容，感謝各位的閱讀！希望分享的內(nèi)容對大家有幫助，更多相關(guān)知識(shí)，歡迎關(guān)注億速云行業(yè)資訊頻道！
免責(zé)聲明：本站發(fā)布的內(nèi)容（圖片、視頻和文字）以原創(chuàng)、轉(zhuǎn)載和分享為主，文章觀點(diǎn)不代表本網(wǎng)站立場，如果涉及侵權(quán)請聯(lián)系站長郵箱：is@yisu.com進(jìn)行舉報(bào)，并提供相關(guān)證據(jù)，一經(jīng)查實(shí)，將立刻刪除涉嫌侵權(quán)內(nèi)容。
億速云公眾號(hào)
手機(jī)網(wǎng)站二維碼