Seq No
|
Deprecated API
|
NEW API
|
1
|
Release 0.20
|
Release 1.X and 2.X
|
2
|
Interfaces (means you can add a method with default implementation to
an abstract class without breaking old implementations of class
|
Abstract classes
For Example Mapper and Reducer interfaces in Old API are abstract
classes in new API
|
3
|
Package org.apache.hadoop.mapred
|
org.apache.hadoop.mapreduce
|
4
|
JobConf,the OutputCollector and the Reporter
|
Context Object (allows the user
code to communicate with MapReduce System)
|
5
|
Both API’s, Key-Value record pairs are pushed to mapper and reducer
|
In addition execution flow can be controlled by run() method
|
6
|
Job Control by JobClient class
|
Job class
|
7
|
Output file are named as Part-nnnnn
|
Part-m-nnnnn( mapper) , part-r-nnnnn(reducer)
|
8
|
No provision
|
User overridable methods are
declared to throw java.lang.InteruptedException
|
9
|
reduce() method passes values to java.lang.Iterator
|
java.lang.Iterable (making easier to iterate over the values using
Java’s for-each loop construct for(VALUEIN value:values){………}
|
10
|
Configuration c = new Configuration;
Job j = new Job (c,”TEST JOB”);
|
jobConf conf = new jobConf();
conf.setJarByClass();
|
11
|
System.exit(job.waitforcompletion(true)?0:1);
|
jobClient.runjob(conf);
|
Deprecated API Sample :
Public class OldMaxTemp {
static class OldMaxTempMapper extends MapReduceBase
implements Mapper<LongWritable,Text,Text,IntWritable> {
private static final int MISSING = 9999;
@Override
public void map(LongWritable key,Text Value,
OutputCollector<Text,IntWritable> output,Reporter reporter)
throws IOException {
String line = value.toString();
String year = line.substring(15,19);
int airTemp;
if (line.charAt(87) == '+')
{ //parse Int doesn't like leading plus signs
airTemp = Integer.parseInt(line.substring(88,92));
} else
{
airTemp = Integer.parseInt(line.substring(87,92));
}
String quality = line.substing(92,93);
if (airTemp != MISSING && quality.matches("[01459]")) {
output.collect(new Text(year) , new IntWritable(airTemp));
}
}
}
static class oldMaxTempReducer extends MapReduceBase
implements Reducer<Text,IntWritable,Text,IntWritable>{
@override
public void reduce(Text Key,Iterator<IntWritable> values,
OutputCollector<Text , IntWritable> output,Reporter reporter)
throws IOException {
int maxValue = Integer.MIN_VALUE;
while (values.hasNext()){
maxValue =Math.max(maxValue,values.next().get());
}
output.collect(key, new IntWritable(maxValue));
}
}
public static void main(Sting[] args throws IOException {
if (args.length !=2) {
System.err.println("Usage:Old Max Temp <input> <output>");
System.exit(-1);
}
JobConf conf = new JobConf(OldMaxTemp.class);
conf.setJobName("MAX TEMP");
FileInputFormat.addInputPath(conf,new Path(args[0]));
FileOutputFormat.setOutputPath(conf,new Path(args[1]));
conf.setMapperClass(OldMaxTempMapper.class);
conf.setReducerClass(OldMaxTempReducer.class);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(IntWritable.class);
JobClient.runJob(conf);
}
}
New API Sample:
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class MaxTempMapper
extends Mapper<LongWritable,Text,Text,IntWritable> {
private static final int MISSING = 9999;
@Override
public void map(LongWritable key,Text value,Context context)
throws IOException,InterruptedException {
String line = value.toString();
String year = line.substring(15,19);
int airTemp;
if (line.cahrAt(87) == '+' {
//parseInt doesn't like leading plus sign
airTemp = Integer.parseInt(line.substring(88,92);
}else {
airTemp = Integer.parseInt(line.substring(87,92);
}
String quality = line.substing(92,93);
if (airTemp != MISSING && quality.matches("[01459]") {
context.write(new Text(year),new IntWritable(airTemp));
}
}
}
publice class MaxTempReducer
extends Reducer<Text,IntWritable,Text,IntWritable>{
@override
public void reduce(Text key,Iterable<IntWritable> values,Context context)
throws IOException, InterruptedException {
int maxValue = Integer.MIN_VALUE;
for (Intwritable value :values) {
maxValue = Math.max(maxValue,value.get());
}
context.write(key , new IntWritable(maxValue));
}
}
public class MaxTemp {
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.err.println("Usage :Max Temp <input> <output>");
System.exit(-1);
}
Job job = new Job();
job.setJarByClass(MaxTemp.class);
job.setJobName("MAX TEMP");
FileInputFormat.addInputPath(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]);
job.setMapperClass(MaxTempMapper.class);
job.setReducerClass(MaxTempReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
System.exit(job.waitForCompletion(true) ? 0:1);
}
}
Comments
Post a Comment