Skip to content

Commit

Permalink
Fix parsing for Deflate specification edge case
Browse files Browse the repository at this point in the history 
Relates to #1.
  • Loading branch information
@NeRdTheNed
NeRdTheNed committed Oct 7, 2023
1 parent a485894 commit e01242b
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 20 deletions.
12 changes: 11 additions & 1 deletion deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,16 @@ public static int distance2dist(long distance) {
: Constants.distance2dist_hi[(int) ((distance - 1) >> 7)];
}

public static int len2litlen(int len, boolean edgecase) {
assert !edgecase || (len == Constants.MAX_LEN);

if (edgecase) {
return 284;
}

return len2litlen[len];
}

public static final int LZZ_BACKREF_LEN = 32 * 1024;

public static final int LITLEN_MAX = 285;
Expand Down Expand Up @@ -117,7 +127,7 @@ public static int distance2dist(long distance) {
/* 29 */ new DistancePair( 24577, 13 ),
};

static final int[] len2litlen = {
private static final int[] len2litlen = {
/* 0 */ 0xffff,
/* 1 */ 0xffff,
/* 2 */ 0xffff,
Expand Down
28 changes: 14 additions & 14 deletions deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateBlockHuffman.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ private static int getLitLenSize(LitLen litlenThis, Huffman litlenDec, Huffman d
if (litlenThis.dist > 0L) {
final long distance = litlenThis.dist;
final long len = litlenThis.litlen;
final long litlen = Constants.len2litlen[(int) len];
final long litlen = Constants.len2litlen((int) len, litlenThis.edgecase);
// litlen bits
long nbits = litlenDec.getSymLen((int) litlen);
// ebits
Expand Down Expand Up @@ -384,7 +384,7 @@ public void removeDistLitLeastExpensive(int mode) {
checkNext: for (final LitLen check : litlens) {
if (check.dist != 0L) {
assert check.decodedVal != null;
final int litlen = Constants.len2litlen[(int) check.litlen] - Constants.LITLEN_TBL_OFFSET;
final int litlen = Constants.len2litlen((int) check.litlen, check.edgecase) - Constants.LITLEN_TBL_OFFSET;
litSeen[litlen] = true;

if (litNoAllow[litlen]) {
Expand Down Expand Up @@ -436,7 +436,7 @@ public void removeDistLitLeastExpensive(int mode) {

if (check.dist != 0L) {
assert check.decodedVal != null;
final int litlen = Constants.len2litlen[(int) check.litlen] - Constants.LITLEN_TBL_OFFSET;
final int litlen = Constants.len2litlen((int) check.litlen, check.edgecase) - Constants.LITLEN_TBL_OFFSET;

if (litlen != litlenRem) {
continue;
Expand Down Expand Up @@ -673,7 +673,7 @@ public void recodeHuffman() {

for (final LitLen litlenThis : litlens) {
if (litlenThis.dist > 0L) {
litFreqTemp[Constants.len2litlen[(int) litlenThis.litlen]]++;
litFreqTemp[Constants.len2litlen((int) litlenThis.litlen, litlenThis.edgecase)]++;
distFreqTemp[Constants.distance2dist(litlenThis.dist)]++;
} else {
litFreqTemp[(int) litlenThis.litlen]++;
Expand Down Expand Up @@ -837,13 +837,13 @@ private boolean decodeStream(BitInputStream is) throws IOException {
}

assert (len >= Constants.MIN_LEN) && (len <= Constants.MAX_LEN);
// The Deflate specification doesn't specify if distance codes with length 258 using code 284 are allowed,
// but they're not explicitly forbidden.
// See https://github.com/NeRdTheNed/deft4j/issues/1
final boolean edgecase = (len == Constants.MAX_LEN) && (litlen == 284);

// TODO: Proper fix, see https://github.com/NeRdTheNed/deft4j/issues/1
// The Deflate specification doesn't specify if distance codes with length 258 using code 284 are allowed.
// deft4j doesn't currently handle such codes correctly.
if ((len == Constants.MAX_LEN) && (litlen == 284)) {
System.out.println("deft4j issue: Deflate specification edge case encountered when parsing input file.");
return false;
if (DEBUG_PRINT_PARSE && edgecase) {
System.out.println("Edge case: code 284 used for size 258");
}

// Get the distance
Expand All @@ -865,7 +865,7 @@ private boolean decodeStream(BitInputStream is) throws IOException {
}

assert (dist >= Constants.MIN_DISTANCE) && (dist <= Constants.MAX_DISTANCE);
final LitLen readLen = new LitLen(dist, len);
final LitLen readLen = new LitLen(dist, len, edgecase);
sizeBits += totalSize;
litlenSizeBits += totalSize;

Expand Down Expand Up @@ -1107,9 +1107,9 @@ private void writeSym(BitOutputStream os, long val) throws IOException {
os.writeNBits(litlenDec.getSym((int) val), litlenDec.getSymLen((int) val));
}

private void writeBackref(BitOutputStream os, long len, long distance) throws IOException {
private void writeBackref(BitOutputStream os, long len, long distance, boolean edgecase) throws IOException {
// Back reference length
final long litlen = Constants.len2litlen[(int) len];
final long litlen = Constants.len2litlen((int) len, edgecase);
// litlen bits
long bits = litlenDec.getSym((int) litlen);
long nbits = litlenDec.getSymLen((int) litlen);
Expand All @@ -1133,7 +1133,7 @@ private void writeLitLen(BitOutputStream os, LitLen litlenThis) throws IOExcepti
if (litlenThis.dist == 0L) {
writeSym(os, litlenThis.litlen);
} else {
writeBackref(os, litlenThis.litlen, litlenThis.dist);
writeBackref(os, litlenThis.litlen, litlenThis.dist, litlenThis.edgecase);
}
}

Expand Down
16 changes: 11 additions & 5 deletions deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/LitLen.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ public int hashCode() {
final int prime = 31;
int result = 1;
result = (prime * result) + Arrays.hashCode(decodedVal);
return (prime * result) + Objects.hash(dist, litlen);
return (prime * result) + Objects.hash(dist, edgecase, litlen);
}

@Override
Expand All @@ -23,25 +23,31 @@ public boolean equals(Object obj) {
}

final LitLen other = (LitLen) obj;
return Arrays.equals(decodedVal, other.decodedVal) && (dist == other.dist) && (litlen == other.litlen);
return Arrays.equals(decodedVal, other.decodedVal) && (dist == other.dist) && (edgecase == other.edgecase) && (litlen == other.litlen);
}

public final long dist;
public final long litlen;
public final boolean edgecase;

public byte[] decodedVal;

public LitLen(long dist, long litlen) {
public LitLen(long dist, long litlen, boolean edgecase) {
this.dist = dist;
this.litlen = litlen;
this.edgecase = edgecase;
}

public LitLen(long dist, long litlen) {
this(dist, litlen, false);
}

public LitLen(long litlen) {
this(0, litlen);
}

public LitLen copy() {
final LitLen newLit = new LitLen(dist, litlen);
final LitLen newLit = new LitLen(dist, litlen, edgecase);

if (decodedVal != null) {
//newLit.decodedVal = new byte[decodedVal.length];
Expand All @@ -54,6 +60,6 @@ public LitLen copy() {

@Override
public String toString() {
return "LitLen [litlen=" + litlen + ", dist=" + dist + ", decodedVal=" + Arrays.toString(decodedVal) + "]";
return "LitLen [litlen=" + litlen + ", dist=" + dist + ", edgecase=" + edgecase + ", decodedVal=" + Arrays.toString(decodedVal) + "]";
}
}

0 comments on commit e01242b

Please sign in to comment.