hadoop2x WordCount MapReduce怎么用

這篇文章主要介紹了hadoop2x WordCount  MapReduce怎么用，具有一定借鑒價值，感興趣的朋友可以參考下，希望大家閱讀完這篇文章之后大有收獲，下面讓小編帶著大家一起了解一下。

package com.jhl.haoop.examples;

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.util.GenericOptionsParser;

public class WordCount {

// map區(qū)域

public static class TokenizerMapper extends

Mapper<LongWritable, Text, Text, IntWritable> {

private final static IntWritable one = new IntWritable(1);//每個單詞統(tǒng)計一次

private Text word = new Text();

public void map(LongWritable key, Text value, Context context)

throws IOException, InterruptedException {

//進(jìn)行分割 [空格 制表符 \t 換行 \n 回車符\r \f]

// public StringTokenizer(String str) {

//this(str, " \t\n\r\f", false);

       // }

StringTokenizer itr = new StringTokenizer(value.toString());//獲取每行數(shù)據(jù)的值value.toString()

while (itr.hasMoreTokens()) {

word.set(itr.nextToken());//設(shè)置map輸出的key值

context.write(word, one);//上下文輸出map的key和value值 

}

}

}

//reduce 區(qū)域

public static class IntSumReducer extends

Reducer<Text, IntWritable, Text, IntWritable> {

private IntWritable result = new IntWritable();

public void reduce(Text key, Iterable<IntWritable> values,

Context context) throws IOException, InterruptedException {

int sum = 0;

for (IntWritable val : values) {//循環(huán)遍歷Iterable

sum += val.get();//累加

}

result.set(sum);//設(shè)置總次數(shù)

context.write(key, result);

}

}

   //client區(qū)域

public static void main(String[] args) throws Exception {

Configuration conf = new Configuration();//獲取配置信息

//GenericOptionsParser 用來常用的Hadoop命令選項，并根據(jù)需要，為Configuration對象設(shè)置相應(yīng)的取值。

String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

if (otherArgs.length != 2) {

System.err.println("Usage: wordcount  ");

System.exit(2);

}

Job job = new Job(conf, "WordCount");//創(chuàng)建Job、設(shè)置Job配置和名稱

job.setJarByClass(WordCount.class);//設(shè)置Job 運(yùn)行的類

job.setMapperClass(TokenizerMapper.class);//設(shè)置Mapper類和Reducer類

job.setCombinerClass(IntSumReducer.class);

job.setReducerClass(IntSumReducer.class);

FileInputFormat.addInputPath(job, new Path(otherArgs[0]));//設(shè)置輸入文件的路徑和輸出文件的路徑

FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

job.setOutputKeyClass(Text.class);//設(shè)置輸出結(jié)果的key和value類型

job.setOutputValueClass(IntWritable.class);

boolean isSuccess = job.waitForCompletion(true);//提交Job,等待運(yùn)行結(jié)果，并在客戶端顯示運(yùn)行信息

System.exit(isSuccess ? 0 : 1);//結(jié)束程序

}

}

感謝你能夠認(rèn)真閱讀完這篇文章，希望小編分享的“hadoop2x WordCount  MapReduce怎么用”這篇文章對大家有幫助，同時也希望大家多多支持億速云，關(guān)注億速云行業(yè)資訊頻道，更多相關(guān)知識等著你來學(xué)習(xí)!