fixing the compatibility tests

parquet-compat-1.0.0/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>parquet</groupId>
   <artifactId>parquet-compat-1.0.0</artifactId>
-  <version>1.0.0-SNAPSHOT</version>
+  <version>1.0.0</version>
   <packaging>jar</packaging>
 
   <name>Parquet Compatibility 1.0.0</name>
@@ -14,18 +14,30 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
   </properties>
 
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <argLine>-Xmx1024m</argLine>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
   <dependencies>
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>parquet-column</artifactId>
-      <version>1.0.0-SNAPSHOT</version>
+      <version>1.0.0</version>
       <scope>test</scope>
     </dependency>
 
     <dependency>
       <groupId>com.twitter</groupId>      
       <artifactId>parquet-hadoop</artifactId>
-      <version>1.0.0-SNAPSHOT</version>
+      <version>1.0.0</version>
       <scope>test</scope>
     </dependency>
 

parquet-compat-1.1.0/pom.xml

@@ -0,0 +1,51 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>parquet</groupId>
+  <artifactId>parquet-compat-1.1.0</artifactId>
+  <version>1.1.0</version>
+  <packaging>jar</packaging>
+
+  <name>Parquet Compatibility 1.1.0</name>
+  <url>https://github.com/Parquet/parquet-compatibility</url>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <argLine>-Xmx1024m</argLine>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.twitter</groupId>
+      <artifactId>parquet-column</artifactId>
+      <version>1.1.0</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>com.twitter</groupId>      
+      <artifactId>parquet-hadoop</artifactId>
+      <version>1.1.0</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.10</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>

parquet-compat-1.1.0/src

@@ -0,0 +1 @@
+../parquet-compat/src

parquet-compat/src/test/java/parquet/compat/test/ConvertUtils.java

@@ -100,10 +100,11 @@ public static void convertCsvToParquet(File csvFile, File outputParquetFile, boo
         writer.write(Arrays.asList(fields));
         ++lineNumber;
       }
+
+      writer.close();
     } finally {
       LOG.info("Number of lines: " + lineNumber);
       Utils.closeQuietly(br);
-      Utils.closeQuietly(writer);
     } 
   }
 
@@ -134,9 +135,9 @@ public static void convertParquetToCSV(File parquetFile, File csvOutputFile) thr
       while( (g = reader.read())!= null) {
         writeGroup(w, g, schema);
       }
+      reader.close();
     }
     finally {
-      Utils.closeQuietly(reader);
       Utils.closeQuietly(w);
     }
   }

parquet-compat/src/test/java/parquet/compat/test/TestPerfRegression.java

@@ -27,37 +27,59 @@
 public class TestPerfRegression {
   private static final Log LOG = Log.getLog(TestPerfRegression.class);
 
+  private static final int fileSizeMegs = 1000;
+
   @Test
   public void testWritePerf() throws IOException {
     // With no dictionary - default
-    File csvTestFile = Utils.createTestFile(1000);
+    File csvTestFile = Utils.createTestFile(fileSizeMegs);
 
-    File parquetTestFile = Utils.getParquetOutputFile("perf", "1000", true);
+    File parquetTestFile = Utils.getParquetOutputFile("perf", String.valueOf(fileSizeMegs), true);
     long startTime = System.currentTimeMillis();
     ConvertUtils.convertCsvToParquet(csvTestFile, parquetTestFile);
     long endTime = System.currentTimeMillis();
 
     long totalTime = (endTime - startTime);
     LOG.info("Write Time: " + totalTime ); 
 
-    assertTrue(totalTime < 45000);
+    Utils.writePerfResult("write", totalTime);
+
+    LOG.info("Time taken to write " + fileSizeMegs + " sized csv file : " + fileSizeMegs);
+
+    assertTrue(totalTime < 60000);
+
+    // It should not be slower than previous versions
+    for(String version : Utils.getAllPreviousVersionDirs()) {
+      long totalTimeForVersion = Utils.readPerfResult(version, "write");
+      LOG.info("Time taken to write with version "+ version + ": " + totalTimeForVersion);
+      assertTrue(totalTime < 1.1 * totalTimeForVersion);
+    }
   }
 
   @Test
   public void testReadPerf() throws IOException {
-    File parquetTestFile = Utils.getParquetOutputFile("perf", "1000", false);
+    File parquetTestFile = Utils.getParquetOutputFile("perf", String.valueOf(fileSizeMegs), false);
     if(!parquetTestFile.exists()) {
       throw new IOException("File "+ parquetTestFile.getName() + " does not exists, Run testWritePerf");
     }
-    File csvTestFile = Utils.getCsvTestFile("perf", "1000", true);
+    File csvTestFile = Utils.getCsvTestFile("perf", String.valueOf(fileSizeMegs), true);
     long startTime = System.currentTimeMillis();
     ConvertUtils.convertParquetToCSV(parquetTestFile, csvTestFile );
     long endTime = System.currentTimeMillis();
 
     long totalTime = (endTime - startTime);
     LOG.info("Read Time: " + totalTime ); 
 
+    Utils.writePerfResult("read", totalTime);
+
     assertTrue(totalTime < 45000);
+
+    // It should not be slower than previous versions
+    for(String version : Utils.getAllPreviousVersionDirs()) {
+      long totalTimeForVersion = Utils.readPerfResult(version, "read");
+      LOG.info("Time taken to read with version "+ version + ": " + totalTimeForVersion);
+      assertTrue(totalTime < 1.1 * totalTimeForVersion);
+    }
   }
 
 }

parquet-compat/src/test/java/parquet/compat/test/TestUtils.java

@@ -0,0 +1,28 @@
+package parquet.compat.test;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+public class TestUtils {
+
+  @Test
+  public void testVersionComparator() {
+    Utils.Version v1 = new Utils.Version("1.0.0");
+    Utils.Version v2 = new Utils.Version("1.0.1");
+
+    Assert.assertTrue(v1.compareMajorMinor(v2)==0);
+    Assert.assertTrue(v1.compareTo(v2) < 0);
+
+    v2 = new Utils.Version("1.1.0");
+
+    Assert.assertTrue(v1.compareMajorMinor(v2) < 0);
+    Assert.assertTrue(v1.compareTo(v2) < 0);
+
+    v2 = new Utils.Version("1.0.0-SNAPSHOT");
+
+    Assert.assertTrue(v1.compareTo(v2) > 0);
+
+  }
+
+}

parquet-compat/src/test/java/parquet/compat/test/Utils.java

@@ -38,6 +38,7 @@
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.io.file.tfile.Utils.Version;
 
 import parquet.Log;
 
@@ -55,6 +56,29 @@ public static void closeQuietly(Closeable res) {
     }    
   }
 
+  public static void writePerfResult(String module, long millis) throws IOException{
+    PrintWriter writer = null;
+    try {
+      File outputFile = new File("target/test/perftime." + module + ".txt");
+      outputFile.delete();
+      writer = new  PrintWriter(outputFile);
+      writer.write(String.valueOf(millis));
+    } finally {
+      closeQuietly(writer);
+    }
+  }
+
+  public static long readPerfResult(String version, String module) throws IOException {
+    BufferedReader reader = null;
+    try {
+      File inFile = new File("../" + version + "/target/test/perftime." + module + ".txt");
+      reader = new  BufferedReader(new FileReader(inFile));
+      return Long.parseLong(reader.readLine());
+    } finally {
+      closeQuietly(reader);
+    }
+  }
+
   public static File createTestFile(long largerThanMB) throws IOException {
     File outputFile = new File("target/test/csv/perftest.csv");
     if(outputFile.exists()) {
@@ -95,16 +119,103 @@ public boolean accept(File dir, String name) {
 
   public static String[] getAllPreviousVersionDirs() throws IOException {
     File baseDir = new File("..");
-    final String currentVersion = new File(".").getCanonicalFile().getName();
+    final String currentVersion = getCurrentVersion();
     final String[] versions = baseDir.list(new FilenameFilter() {
       public boolean accept(File dir, String name) {
         return name.startsWith("parquet-compat-") 
-            && name.compareTo(currentVersion) <= 0;
+            && new Version(name.replace("parquet-compat-", "")).compareTo(new Version(currentVersion)) < 0;
       }
     });
     return versions;
   }
+
+  static class Version implements Comparable<Version> {
+    int major;
+    int minor;
+    int minorminor;
+    String tag;
+
+    Version(String versionStr) {
+      String[] versions = versionStr.split("\\.");
+      int size = versions.length;
+      if (size > 0) {
+        this.major = Integer.parseInt(versions[0]);
+      }
+      if (size > 1) {
+        this.minor = Integer.parseInt(versions[1]);
+      }
+      if (size > 2) {
+        if(versions[2].contains("-")) {
+          String[] minorMin = versions[2].split("-");
+          this.minorminor = Integer.parseInt(minorMin[0]);
+          this.tag = minorMin[1];
+        } else {
+          this.minorminor = Integer.parseInt(versions[2]);
+        }
+      }
+      if(size == 4) {
+        this.tag = versions[3];
+      }
+      if (size > 4) {
+        throw new RuntimeException("Illegal version number " + versionStr);
+      }
+    }
+
+    public int compareMajorMinor(Version o) {
+      return ComparisonChain.
+          start().
+      compare(major, o.major).
+      compare(minor, o.minor).
+      result();
+    }
 
+    @Override
+    public int compareTo(Version o) {
+      return ComparisonChain.
+          start().
+      compare(major, o.major).
+      compare(minor, o.minor).
+      compare(minorminor, o.minorminor).
+      compare(tag, o.tag).
+      result();
+    }
+
+    // Very basic implementation of comparisonchain
+    private static class ComparisonChain {
+      int result = 0;
+      private ComparisonChain(int result) {
+        this.result = result; 
+      }
+      static ComparisonChain start() {
+        return new ComparisonChain(0);
+      }
+      ComparisonChain compare(String a, String b) {
+        if (result != 0) {
+          return this;
+        }
+        if(b == null) {
+          if (a!= null) result=1;
+          else result= 0;
+        } else if (a == null) {
+          result=1;
+        } else if (result == 0) {
+          result = a.compareTo(b);
+        }
+        return this;
+      }
+      ComparisonChain compare(int a, int b) {
+        if (result == 0) {
+          result = Integer.compare(a, b);
+        }
+        return this;
+      }
+      int result() {
+        return result;
+      }
+    }    
+  }
+
+
   public static File getParquetOutputFile(String name, String module, boolean deleteIfExists) {
     File outputFile = new File("target/parquet/", getParquetFileName(name, module));
     outputFile.getParentFile().mkdirs();
@@ -132,11 +243,22 @@ public static File getParquetFile(String name, String version, String module, bo
     return parquetFile;
   }
 
-  public static String[] getImpalaDirectories() {
+  private static String getCurrentVersion() throws IOException {
+    return new File(".").getCanonicalFile().getName().replace("parquet-compat-", "");
+  }
+
+  public static String[] getImpalaDirectories() throws IOException {
     File baseDir = new File("../parquet-testdata/impala");
+    final String currentVersion = getCurrentVersion();
     final String[] impalaVersions = baseDir.list(new FilenameFilter() {
       public boolean accept(File dir, String name) {
-        return !name.startsWith(".");
+        if (name.startsWith(".")) {
+          return false;
+        }
+        if (name.contains("-")) {
+          name = name.split("-")[0];
+        }
+        return new Version(name).compareMajorMinor(new Version(currentVersion)) == 0;
       }
     });
     return impalaVersions;