Final fixes for demo

Author: Josh Devins

README.md

@@ -21,3 +21,18 @@ Unit testing of Pig is currently done using PigUnit, a new xUnit style testing h
  * heap space is at a premium when running the Pig scripts, so max heap space JVM parameters need to be set (to something like`-Xmx1024m`)
    * running tests from Maven: configured in the Maven `pom` to run the Surefire plugin with a fixed max heap space setting
    * running tests from Eclipse: use the [JUnit Lanch Fixer](http://code.google.com/p/junitlaunchfixer) plugin and set the max heap space for all JUnit executions automatically (note that if you have previous failed launches, you should delete them before running again)
+
+Running
+---
+
+Checkout the source and build it with the Maven assembly plugin.
+
+    mvn assembly:assembly
+
+The demo is done using the Cloudera [training VM v0.3.5](http://cloudera-vm.s3.amazonaws.com/cloudera-demo-0.3.5.tar.bz2?downloads) with CDH3b3. Here are the steps to run the demo. This assumes that you move/copy/mount the Git directory/checkout onto the VM.
+
+    hadoop fs -rmr access-log-throughput-mr; hadoop jar target/hadoop-getting-started-1-SNAPSHOT-jar-with-dependencies.jar net.joshdevins.talks.hadoopstart.mr.AccessLogThroughputDriver
+    hadoop fs -cat access-log-throughput-mr/part-* | more
+
+    hadoop fs -rmr access-log-throughput; java -cp target/hadoop-getting-started-1-SNAPSHOT-jar-with-dependencies.jar:/etc/hadoop/conf org.apache.pig.Main -logfile target/pig.log src/main/pig/access-log-throughput.pig
+    hadoop fs -cat access-log-throughput/part-* | more

src/main/java/net/joshdevins/talks/hadoopstart/mr/AccessLogThroughputDriver.java

@@ -20,9 +20,8 @@ public class AccessLogThroughputDriver extends Configured implements Tool {
     @Override
     public int run(final String[] args) throws Exception {
 
-        if (args.length != 2) {
-            System.err.printf("Usage: %s [generic options] <input> <output>\n", AccessLogThroughputDriver.class
-                    .getSimpleName()); // NOPMD
+        if (args.length != 0) {
+            System.err.printf("Usage: %s [generic options]\n", AccessLogThroughputDriver.class.getSimpleName()); // NOPMD
             ToolRunner.printGenericCommandUsage(System.err);
             return -1;
         }
@@ -31,8 +30,8 @@ public int run(final String[] args) throws Exception {
         job.setJarByClass(AccessLogThroughputDriver.class);
 
         // set the input and output paths
-        FileInputFormat.setInputPaths(job, args[0]);
-        FileOutputFormat.setOutputPath(job, new Path(args[1]));
+        FileInputFormat.setInputPaths(job, "data/logs/*-access.log");
+        FileOutputFormat.setOutputPath(job, new Path("access-log-throughput-mr"));
 
         // submit job and wait for completion
         boolean success = job.waitForCompletion(true);

src/main/java/net/joshdevins/talks/hadoopstart/mr/ApacheCombinedAccessLogParser.java

@@ -38,7 +38,13 @@ public static ApacheCombinedAccessLogEntry parse(final String raw) {
         entry.setUri(matcher.group(6));
         entry.setProtocol(matcher.group(7));
         entry.setStatusCode(Integer.parseInt(matcher.group(8)));
-        entry.setBytes(Long.parseLong(matcher.group(9)));
+
+        try {
+            entry.setBytes(Long.parseLong(matcher.group(9)));
+        } catch (NumberFormatException nfe) {
+            // just ignore this and don't set the bytes
+        }
+
         entry.setReferrer(matcher.group(10));
         entry.setUserAgent(matcher.group(11));
 

src/main/pig/access-log-throughput.pig

@@ -1,6 +1,8 @@
 set job.name 'Apache Access Log Throughput'
 set default_parallel 2
 
+REGISTER target/hadoop-getting-started-1-SNAPSHOT-jar-with-dependencies.jar
+
 -- PiggyBank UDFs
 DEFINE ApacheCombinedLogLoader org.apache.pig.piggybank.storage.apachelog.CombinedLogLoader();
 DEFINE SUBSTRING org.apache.pig.piggybank.evaluation.string.SUBSTRING();

src/test/java/net/joshdevins/talks/hadoopstart/mr/ApacheCombinedAccessLogParserTest.java

@@ -30,9 +30,18 @@ public void testParseValidLine() throws Exception {
         Assert.assertEquals(200, entry.getStatusCode());
         Assert.assertEquals(3190L, entry.getBytes());
         Assert.assertEquals("-", entry.getReferrer());
-        Assert
-                .assertEquals(
-                        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4; en-us) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1",
-                        entry.getUserAgent());
+        Assert.assertEquals(
+                "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4; en-us) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1",
+                entry.getUserAgent());
+    }
+
+    @Test
+    public void testParseValidLine_NoBytes() {
+
+        ApacheCombinedAccessLogEntry entry = ApacheCombinedAccessLogParser
+                .parse("1.2.3.4 - - [30/Sep/2008:15:07:53 -0400] \"GET / HTTP/1.1\" 200 - \"-\" \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4; en-us) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1\"");
+
+        Assert.assertNotNull(entry);
+        Assert.assertEquals(0L, entry.getBytes());
     }
 }