Compute max compressed length for SkippableLongCODEC

Author: piotrrzysko

src/main/java/me/lemire/longcompression/LongBinaryPacking.java

@@ -23,8 +23,9 @@
  * @author Benoit Lacelle
  */
 public final class LongBinaryPacking implements LongCODEC, SkippableLongCODEC {
-        final static int BLOCK_SIZE = 64;
-    
+        public final static int BLOCK_SIZE = 64;
+        private static final int MAX_BIT_WIDTH = Long.SIZE;
+
         @Override
         public void compress(long[] in, IntWrapper inpos, int inlength,
                 long[] out, IntWrapper outpos) {
@@ -136,6 +137,15 @@ public void headlessUncompress(long[] in, IntWrapper inpos, int inlength,
             inpos.set(tmpinpos);
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+            int blockCount = inlength / BLOCK_SIZE;
+            int headersSizeInLongs = blockCount / Long.BYTES + (blockCount % Long.BYTES);
+            int blocksSizeInLongs = blockCount * MAX_BIT_WIDTH;
+            compressedPositions.add(blockCount * BLOCK_SIZE);
+            return headersSizeInLongs + blocksSizeInLongs;
+        }
+
         @Override
         public String toString() {
                 return this.getClass().getSimpleName();

src/main/java/me/lemire/longcompression/LongCompressor.java

@@ -40,7 +40,8 @@ public LongCompressor() {
      * @throws UncompressibleInputException if the data is too poorly compressible
      */
     public  long[] compress(long[] input) {
-    	long[] compressed = new long[input.length + input.length / 100 + 1024];
+        int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), input.length);
+        long[] compressed = new long[maxCompressedLength + 1]; // +1 to store the length of the input
         // Store at index=0 the length of the input, hence enabling .headlessCompress
         compressed[0] = input.length;
         IntWrapper outpos = new IntWrapper(1);

src/main/java/me/lemire/longcompression/LongJustCopy.java

@@ -43,6 +43,12 @@ public void headlessUncompress(long[] in, IntWrapper inpos, int inlength,
 
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+            compressedPositions.add(inlength);
+            return inlength;
+        }
+
         @Override
         public void compress(long[] in, IntWrapper inpos, int inlength,
                 long[] out, IntWrapper outpos) {

src/main/java/me/lemire/longcompression/LongVariableByte.java

@@ -22,6 +22,7 @@
  * @author Benoit Lacelle
  */
 public class LongVariableByte implements LongCODEC, ByteLongCODEC, SkippableLongCODEC {
+    private static final int MAX_BYTES_PER_INT = 10;
 
     private static byte extract7bits(int i, long val) {
         return (byte) ((val >>> (7 * i)) & ((1 << 7) - 1));
@@ -326,6 +327,14 @@ public void headlessUncompress(long[] in, IntWrapper inpos, int inlength, long[]
         inpos.set(p + (s!=0 ? 1 : 0));
     }
 
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+        int maxLengthInBytes = inlength * MAX_BYTES_PER_INT;
+        int maxLengthInLongs = (maxLengthInBytes + Long.BYTES - 1) / Long.BYTES;
+        compressedPositions.add(inlength);
+        return maxLengthInLongs;
+    }
+
     /**
      * Creates a new buffer of the requested size.
      *

src/main/java/me/lemire/longcompression/SkippableLongCODEC.java

@@ -67,4 +67,21 @@ public void headlessCompress(long[] in, IntWrapper inpos, int inlength, long[] o
     public void headlessUncompress(long[] in, IntWrapper inpos, int inlength, long[] out,
             IntWrapper outpos, int num);
 
+    /**
+     * Compute the maximum number of longs that might be required to store
+     * the compressed form of a given input array segment, without headers.
+     * <p>
+     * This is useful to pre-allocate the output buffer before calling
+     * {@link #headlessCompress(long[], IntWrapper, int, long[], IntWrapper)}.
+     * </p>
+     *
+     * @param compressedPositions
+     *        since not all schemes compress every input integer, this parameter
+     *        returns how many input integers will actually be compressed.
+     *        This is useful when composing multiple schemes.
+     * @param inlength
+     *            number of longs to be compressed
+     * @return the maximum number of longs needed in the output array
+     */
+    int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength);
 }

src/main/java/me/lemire/longcompression/SkippableLongComposition.java

@@ -62,6 +62,16 @@ public void headlessUncompress(long[] in, IntWrapper inpos, int inlength, long[]
         F2.headlessUncompress(in, inpos, inlength, out, outpos, num);
     }
 
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+        int init = compressedPositions.get();
+        int maxLength = F1.maxHeadlessCompressedLength(compressedPositions, inlength);
+        maxLength += 1; // Add +1 for the potential F2 header. Question: is this header actually needed in the headless version?
+        inlength -= compressedPositions.get() - init;
+        maxLength += F2.maxHeadlessCompressedLength(compressedPositions, inlength);
+        return maxLength;
+    }
+
     @Override
     public String toString() {
         return F1.toString() + "+" + F2.toString();

src/test/java/me/lemire/longcompression/LongTestUtils.java

@@ -111,7 +111,7 @@ protected static long[] uncompress(ByteLongCODEC codec, byte[] data, int len) {
     }
 
     protected static long[] compressHeadless(SkippableLongCODEC codec, long[] data) {
-        long[] outBuf = new long[data.length * 4];
+        long[] outBuf = new long[codec.maxHeadlessCompressedLength(new IntWrapper(0), data.length)];
         IntWrapper inPos = new IntWrapper();
         IntWrapper outPos = new IntWrapper();
         codec.headlessCompress(data, inPos, data.length, outBuf, outPos);

src/test/java/me/lemire/longcompression/SkippableLongBasicTest.java

@@ -15,6 +15,7 @@
 import me.lemire.integercompression.TestUtils;
 import me.lemire.integercompression.VariableByte;
 
+import static org.junit.Assert.assertTrue;
 
 /**
  * Just some basic sanity tests.
@@ -42,10 +43,11 @@ public void consistentTest() {
         for (SkippableLongCODEC c : codecs) {
             System.out.println("[SkippeableBasicTest.consistentTest] codec = "
                     + c);
-            long[] outBuf = new long[N + 1024];
             for (int n = 0; n <= N; ++n) {
                 IntWrapper inPos = new IntWrapper();
                 IntWrapper outPos = new IntWrapper();
+                long[] outBuf = new long[c.maxHeadlessCompressedLength(new IntWrapper(0), n)];
+
                 c.headlessCompress(data, inPos, n, outBuf, outPos);
 
                 IntWrapper inPoso = new IntWrapper();
@@ -142,5 +144,27 @@ public void varyingLengthTest2() {
         }
     }
 
+    @Test
+    public void testMaxHeadlessCompressedLength() {
+        testMaxHeadlessCompressedLength(new LongJustCopy(), 128);
+        testMaxHeadlessCompressedLength(new LongBinaryPacking(), 16 * LongBinaryPacking.BLOCK_SIZE);
+        testMaxHeadlessCompressedLength(new LongVariableByte(), 128);
+        testMaxHeadlessCompressedLength(new SkippableLongComposition(new LongBinaryPacking(), new LongVariableByte()), 16 * LongBinaryPacking.BLOCK_SIZE + 10);
+    }
+
+    private static void testMaxHeadlessCompressedLength(SkippableLongCODEC codec, int inlengthTo) {
+        for (int inlength = 0; inlength < inlengthTo; ++inlength) {
+            long[] input = new long[inlength];
+            Arrays.fill(input, -1L);
 
+            int maxOutputLength = codec.maxHeadlessCompressedLength(new IntWrapper(), inlength);
+            long[] output = new long[maxOutputLength];
+            IntWrapper outPos = new IntWrapper();
+
+            codec.headlessCompress(input, new IntWrapper(), inlength, output, outPos);
+            // If we reach this point, no exception was thrown, which means the calculated output length was sufficient.
+
+            assertTrue(maxOutputLength <= outPos.get() + 1); // +1 because SkippableLongComposition always adds one extra integer for the potential header
+        }
+    }
 }