Skip to content

Commit 840f082

Browse files
umermansoorumermansoor
committed
first commit
0 parents  commit 840f082

16 files changed

Lines changed: 1456 additions & 0 deletions

File tree

‎README.md‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
## Hadoop Example in Java
2+
3+
** Get up and running in less than 5 minutes **
4+
5+
Tis program demonstrates using Hadoop's Map-Reduce concept in Java. The input is a raw data file listing earthquakes by region, magnitude and other information. The goal is to find the maximum magnitude of earthquake for a given region.
6+
7+
### Instructions for Setting Up Hadoop
8+
1. Download Hadoop 1.1.1 binary [http://mirror.csclub.uwaterloo.ca/apache/hadoop/common/hadoop-1.1.1/hadoop-1.1.1.tar.gz]
9+
2. Extract it to a folder on your computer [tar xvfz hadoop-1.1.1.tar.gz].
10+
3. Setup the JAVA_HOME environment variable to point to the directory where Java is installed. For my Mac OSX, I did the following:
11+
$ export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/1.6.0/Home
12+
4. Setup the HADOOP_INSTALL environment variable to point the directory where you extracted hadoop binary in step 2:
13+
$ export HADOOP_INSTALL=/Users/umermansoor/Documents/hadoop-1.1.1
14+
5. Edit the PATH environment variable:
15+
$ export PATH=$PATH:$HADOOP_INSTALL/bin
16+
17+
18+
### Instructions for Running the Sample
19+
1. Clone the project:
20+
$ git clone [email protected]:umermansoor/hadoop-java-example.git
21+
2. Setup the HADOOP_CLASSPATH environment variable to tell Hadoop where to find the java classes for the sample:
22+
$ export HADOOP_CLASSPATH=target/classes/
23+
3. Change to the project directory:
24+
$ cd hadoop-java-example
25+
4. Build the project:
26+
$ mvn clean install
27+
5. Run the sample:
28+
$ hadoop com.umermansoor.App input/input.csv output
29+
Note: the output will go to the `output/` folder which Hadoop will create when run.
30+
31+
### Common Errors:
32+
1. Exception: java.lang.NoClassDefFoundError
33+
Cause: You didn't setup the HADOOP_CLASSPATH environment variable.
34+
Resolution: Setup the variable:
35+
$ export HADOOP_CLASSPATH=target/classes/
36+
37+
2. Exception: org.apache.hadoop.mapred.FileAlreadyExistsException or 'Output directory output already exists'.
38+
Cause: Output directory already exists. Hadoop requires that the output directory doesn't exists when run.
39+
Resolution: Change the output directory or remove the existing one:
40+
$ hadoop com.umermansoor.App input/input.csv output_new -- Change the output directory
41+
Note: Hadoop failing if the output folder already exists is a good thing: it ensures that you don't accidentally overwrite your previous output.
42+

‎input/.DS_Store‎

6 KB
Binary file not shown.

‎input/input.csv‎

Lines changed: 1134 additions & 0 deletions
Large diffs are not rendered by default.

‎pom.xml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<groupId>com.umermansoor</groupId>
6+
<artifactId>hadoopex</artifactId>
7+
<version>1.0-SNAPSHOT</version>
8+
<packaging>jar</packaging>
9+
10+
<name>hadoopex</name>
11+
<url>http://maven.apache.org</url>
12+
13+
<properties>
14+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
15+
</properties>
16+
17+
<dependencies>
18+
<dependency>
19+
<groupId>junit</groupId>
20+
<artifactId>junit</artifactId>
21+
<version>3.8.1</version>
22+
<scope>test</scope>
23+
</dependency>
24+
25+
<dependency>
26+
<groupId>org.apache.hadoop</groupId>
27+
<artifactId>hadoop-core</artifactId>
28+
<version>1.1.1</version>
29+
</dependency>
30+
31+
</dependencies>
32+
</project>

‎src/main/java/com/umermansoor/App.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package com.umermansoor;
2+
3+
import org.apache.hadoop.fs.Path;
4+
import org.apache.hadoop.io.DoubleWritable;
5+
import org.apache.hadoop.io.Text;
6+
import org.apache.hadoop.mapreduce.Job;
7+
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
8+
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
9+
10+
/**
11+
* Hello world!
12+
*
13+
*/
14+
public class App
15+
{
16+
/**
17+
*
18+
* @param args
19+
* @throws Exception - Bad idea but produces less cluttered code.
20+
*/
21+
public static void main(String[] args) throws Exception {
22+
if (args.length != 2) {
23+
System.err.println("Usage: hadoopex <input path> <output path>");
24+
System.exit(-1);
25+
}
26+
27+
// Create the job specification object
28+
Job job = new Job();
29+
job.setJarByClass(App.class);
30+
job.setJobName("Earthquake Measurment");
31+
32+
// Setup input and output paths
33+
FileInputFormat.addInputPath(job, new Path(args[0]));
34+
FileOutputFormat.setOutputPath(job, new Path(args[1]));
35+
36+
// Set the Mapper and Reducer classes
37+
job.setMapperClass(EarthquakeMapper.class);
38+
job.setReducerClass(EarthquakeReducer.class);
39+
40+
// Specify the type of output keys and values
41+
job.setOutputKeyClass(Text.class);
42+
job.setOutputValueClass(DoubleWritable.class);
43+
44+
// Wait for the job to finish before terminating
45+
System.exit(job.waitForCompletion(true) ? 0 : 1);
46+
}
47+
}

‎src/main/java/com/umermansoor/EarthquakeMapper.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package com.umermansoor;
2+
3+
import org.apache.hadoop.io.DoubleWritable;
4+
import org.apache.hadoop.io.LongWritable;
5+
import org.apache.hadoop.io.Text;
6+
import org.apache.hadoop.mapreduce.Mapper;
7+
8+
import java.io.IOException;
9+
10+
11+
/**
12+
* This is the main Mapper class.
13+
* @author umermansoor
14+
*/
15+
public class EarthquakeMapper extends
16+
Mapper<LongWritable, Text, Text, DoubleWritable>
17+
{
18+
19+
/**
20+
* The `Mapper` function. It receives a line of input from the file,
21+
* extracts `region name` and `earthquake magnitude` from it, which becomes
22+
* the output.
23+
* @param key - The line offset in the file - ignored.
24+
* @param value - This is the line itself.
25+
* @param context - Provides access to the OutputCollector and Reporter.
26+
* @throws IOException
27+
* @throws InterruptedException
28+
*/
29+
@Override
30+
public void map(LongWritable key, Text value, Context context) throws
31+
IOException, InterruptedException {
32+
33+
String[] line = value.toString().split(",", 12);
34+
35+
// Ignore invalid lines
36+
if (line.length != 12) {
37+
System.out.println("- " + line.length);
38+
return;
39+
}
40+
41+
// The output `key` is the name of the region
42+
String outputKey = line[11];
43+
44+
// The output `value` is the magnitude of the earthquake
45+
double outputValue = Double.parseDouble(line[8]);
46+
47+
// Record the output in the Context object
48+
context.write(new Text(outputKey), new DoubleWritable(outputValue));
49+
}
50+
}

‎src/main/java/com/umermansoor/EarthquakeReducer.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package com.umermansoor;
2+
3+
import java.io.IOException;
4+
5+
import org.apache.hadoop.io.DoubleWritable;
6+
import org.apache.hadoop.io.Text;
7+
import org.apache.hadoop.mapreduce.Reducer;
8+
9+
public class EarthquakeReducer extends
10+
Reducer<Text, DoubleWritable, Text, DoubleWritable>
11+
{
12+
13+
/**
14+
* The `Reducer` function. Iterates through all earthquake magnitudes for a
15+
* region to find the maximum value. The output is the region name and the
16+
* maximum value of the magnitude.
17+
* @param key - The name of the region
18+
* @param values - Iterator over earthquake magnitudes in the region
19+
* @param context - Used for collecting output
20+
* @throws IOException
21+
* @throws InterruptedException
22+
*/
23+
@Override
24+
public void reduce(Text key, Iterable<DoubleWritable> values,
25+
Context context) throws IOException, InterruptedException {
26+
27+
// Standard algorithm for finding the max value
28+
double maxMagnitude = Double.MIN_VALUE;
29+
for (DoubleWritable value : values) {
30+
maxMagnitude = Math.max(maxMagnitude, value.get());
31+
}
32+
33+
context.write(key, new DoubleWritable(maxMagnitude));
34+
}
35+
}

‎src/test/java/com/umermansoor/AppTest.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package com.umermansoor;
2+
3+
import junit.framework.Test;
4+
import junit.framework.TestCase;
5+
import junit.framework.TestSuite;
6+
7+
/**
8+
* Unit test for simple App.
9+
*/
10+
public class AppTest
11+
extends TestCase
12+
{
13+
/**
14+
* Create the test case
15+
*
16+
* @param testName name of the test case
17+
*/
18+
public AppTest( String testName )
19+
{
20+
super( testName );
21+
}
22+
23+
/**
24+
* @return the suite of tests being tested
25+
*/
26+
public static Test suite()
27+
{
28+
return new TestSuite( AppTest.class );
29+
}
30+
31+
/**
32+
* Rigourous Test :-)
33+
*/
34+
public void testApp()
35+
{
36+
assertTrue( true );
37+
}
38+
}

‎target/classes/com/umermansoor/App.class‎

1.54 KB
Binary file not shown.

‎target/classes/com/umermansoor/EarthquakeMapper.class‎

2.59 KB
Binary file not shown.

0 commit comments

Comments
 (0)