From 6ea7a93cbc84d261ff26686a82a1a061f96fe707 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Thu, 6 Aug 2026 19:20:17 +0200 Subject: [PATCH 1/3] fix(parquet/variant): validate compound value bounds --- parquet/variant/variant.go | 157 +++++++++++++++++++++++++++++--- parquet/variant/variant_test.go | 14 +++ 2 files changed, 159 insertions(+), 12 deletions(-) diff --git a/parquet/variant/variant.go b/parquet/variant/variant.go index 2eec9602..b19f11ef 100644 --- a/parquet/variant/variant.go +++ b/parquet/variant/variant.go @@ -24,6 +24,7 @@ import ( "fmt" "iter" "maps" + "math" "slices" "strings" "time" @@ -515,17 +516,34 @@ func NewWithMetadata(meta Metadata, value []byte) (Value, error) { } func validateScalarValue(value []byte) error { - if basicTypeFromHeader(value[0]) == BasicShortString { + _, err := validateValue(value) + return err +} + +func validateValue(value []byte) (int, error) { + if len(value) == 0 { + return 0, errors.New("invalid variant value: empty") + } + + switch basicTypeFromHeader(value[0]) { + case BasicShortString: want := 1 + int(value[0]>>basicTypeBits) if len(value) < want { - return fmt.Errorf("invalid variant value: short string requires %d bytes, got %d", want, len(value)) + return 0, fmt.Errorf("invalid variant value: short string requires %d bytes, got %d", want, len(value)) } - return nil - } - if basicTypeFromHeader(value[0]) != BasicPrimitive { - return nil + return want, nil + case BasicObject: + return validateObjectValue(value) + case BasicArray: + return validateArrayValue(value) + case BasicPrimitive: + return validatePrimitiveValue(value) + default: + return 0, fmt.Errorf("invalid variant value: unknown basic type %d", basicTypeFromHeader(value[0])) } +} +func validatePrimitiveValue(value []byte) (int, error) { primitiveType := primitiveTypeFromHeader(value[0]) want := 0 switch primitiveType { @@ -551,21 +569,136 @@ func validateScalarValue(value []byte) error { want = 17 case PrimitiveBinary, PrimitiveString: if len(value) < 5 { - return fmt.Errorf("invalid variant value: %s length prefix requires 5 bytes, got %d", primitiveType, len(value)) + return 0, fmt.Errorf("invalid variant value: %s length prefix requires 5 bytes, got %d", primitiveType, len(value)) } dataLen := uint64(binary.LittleEndian.Uint32(value[1:5])) if dataLen > uint64(len(value)-5) { - return fmt.Errorf("invalid variant value: %s data requires %d bytes, got %d", primitiveType, dataLen, len(value)-5) + return 0, fmt.Errorf("invalid variant value: %s data requires %d bytes, got %d", primitiveType, dataLen, len(value)-5) } - return nil + return 5 + int(dataLen), nil default: - return fmt.Errorf("invalid variant value: unknown primitive type %d", primitiveType) + return 0, fmt.Errorf("invalid variant value: unknown primitive type %d", primitiveType) } if len(value) < want { - return fmt.Errorf("invalid variant value: %s requires %d bytes, got %d", primitiveType, want, len(value)) + return 0, fmt.Errorf("invalid variant value: %s requires %d bytes, got %d", primitiveType, want, len(value)) } - return nil + return want, nil +} + +func validateArrayValue(value []byte) (int, error) { + typeInfo := value[0] >> basicTypeBits + offsetSize := uint8(typeInfo&0b11) + 1 + isLarge := ((typeInfo >> 2) & 0x1) != 0 + + var ( + numElements uint32 + offsetStart uint64 + ) + if isLarge { + if len(value) < 5 { + return 0, fmt.Errorf("invalid variant value: array size requires 5 bytes, got %d", len(value)) + } + numElements = readLEU32(value[1:5]) + offsetStart = 5 + } else { + if len(value) < 2 { + return 0, fmt.Errorf("invalid variant value: array size requires 2 bytes, got %d", len(value)) + } + numElements = uint32(value[1]) + offsetStart = 2 + } + + dataStart := offsetStart + (uint64(numElements)+1)*uint64(offsetSize) + if dataStart > uint64(len(value)) || dataStart > math.MaxUint32 { + return 0, fmt.Errorf("invalid variant value: array offset table ends at %d, got %d bytes", dataStart, len(value)) + } + + offsets := make([]uint32, int(numElements)+1) + for i := range offsets { + pos := offsetStart + uint64(i)*uint64(offsetSize) + offset := readLEU32(value[int(pos) : int(pos)+int(offsetSize)]) + if i == 0 && offset != 0 { + return 0, fmt.Errorf("invalid variant value: array first offset must be zero, got %d", offset) + } + if i > 0 && offset < offsets[i-1] { + return 0, fmt.Errorf("invalid variant value: array offsets are not monotonic") + } + if dataStart+uint64(offset) > uint64(len(value)) || dataStart+uint64(offset) > math.MaxUint32 { + return 0, fmt.Errorf("invalid variant value: array offset %d is out of range", offset) + } + offsets[i] = offset + } + + for i := 0; i < len(offsets)-1; i++ { + start := dataStart + uint64(offsets[i]) + end := dataStart + uint64(offsets[i+1]) + childSize, err := validateValue(value[int(start):int(end)]) + if err != nil { + return 0, fmt.Errorf("invalid variant value: array element %d: %w", i, err) + } + if uint64(childSize) != end-start { + return 0, fmt.Errorf("invalid variant value: array element %d has trailing bytes", i) + } + } + + return int(dataStart + uint64(offsets[len(offsets)-1])), nil +} + +func validateObjectValue(value []byte) (int, error) { + typeInfo := value[0] >> basicTypeBits + offsetSize := uint8(typeInfo&0b11) + 1 + idSize := uint8((typeInfo>>2)&0b11) + 1 + isLarge := ((typeInfo >> 4) & 0x1) != 0 + + var ( + numElements uint32 + elementSize uint64 = 1 + ) + if isLarge { + elementSize = 4 + } + if uint64(len(value)) < 1+elementSize { + return 0, fmt.Errorf("invalid variant value: object size requires %d bytes, got %d", 1+elementSize, len(value)) + } + numElements = readLEU32(value[1 : 1+elementSize]) + + idStart := 1 + elementSize + offsetStart := idStart + uint64(numElements)*uint64(idSize) + dataStart := offsetStart + (uint64(numElements)+1)*uint64(offsetSize) + if dataStart > uint64(len(value)) || dataStart > math.MaxUint32 { + return 0, fmt.Errorf("invalid variant value: object offset table ends at %d, got %d bytes", dataStart, len(value)) + } + + offsets := make([]uint32, int(numElements)+1) + for i := range offsets { + pos := offsetStart + uint64(i)*uint64(offsetSize) + offset := readLEU32(value[int(pos) : int(pos)+int(offsetSize)]) + if i == 0 && offset != 0 { + return 0, fmt.Errorf("invalid variant value: object first offset must be zero, got %d", offset) + } + if i > 0 && offset < offsets[i-1] { + return 0, fmt.Errorf("invalid variant value: object offsets are not monotonic") + } + if dataStart+uint64(offset) > uint64(len(value)) || dataStart+uint64(offset) > math.MaxUint32 { + return 0, fmt.Errorf("invalid variant value: object offset %d is out of range", offset) + } + offsets[i] = offset + } + + for i := 0; i < len(offsets)-1; i++ { + start := dataStart + uint64(offsets[i]) + end := dataStart + uint64(offsets[i+1]) + childSize, err := validateValue(value[int(start):int(end)]) + if err != nil { + return 0, fmt.Errorf("invalid variant value: object field %d: %w", i, err) + } + if uint64(childSize) != end-start { + return 0, fmt.Errorf("invalid variant value: object field %d has trailing bytes", i) + } + } + + return int(dataStart + uint64(offsets[len(offsets)-1])), nil } // New creates a Value by parsing both the metadata and value bytes. diff --git a/parquet/variant/variant_test.go b/parquet/variant/variant_test.go index e8216013..11cbfea3 100644 --- a/parquet/variant/variant_test.go +++ b/parquet/variant/variant_test.go @@ -663,6 +663,20 @@ func TestInvalidPrimitiveValue(t *testing.T) { } } +func TestInvalidCompoundValue(t *testing.T) { + for _, input := range []string{`[1,2]`, `{"value":1}`} { + t.Run(input, func(t *testing.T) { + v, err := variant.ParseJSON(input, false) + require.NoError(t, err) + + value := v.Bytes() + _, err = variant.NewWithMetadata(v.Metadata(), value[:len(value)-1]) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid variant value") + }) + } +} + func TestInvalidObjectAccess(t *testing.T) { v := loadVariant(t, "object_primitive") obj := v.Value().(variant.ObjectValue) From 5db5a5496c019f65d2e1b04ae7ece3818112656a Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Thu, 6 Aug 2026 21:23:28 +0200 Subject: [PATCH 2/3] fix(parquet/variant): validate object layouts safely --- parquet/variant/variant.go | 116 +++++++++++++++++------ parquet/variant/variant_test.go | 160 ++++++++++++++++++++++++++++++-- 2 files changed, 237 insertions(+), 39 deletions(-) diff --git a/parquet/variant/variant.go b/parquet/variant/variant.go index b19f11ef..0ccb331f 100644 --- a/parquet/variant/variant.go +++ b/parquet/variant/variant.go @@ -135,6 +135,7 @@ const ( supportedVersion = 1 maxShortStringSize = 0x3F metadataMaxSizeLimit = 128 * 1024 * 1024 // 128MB + maxValidationDepth = 256 ) var ( @@ -508,22 +509,31 @@ func NewWithMetadata(meta Metadata, value []byte) (Value, error) { if len(value) == 0 { return Value{}, errors.New("invalid variant value: empty") } - if err := validateScalarValue(value); err != nil { + if err := validateScalarValue(meta, value); err != nil { return Value{}, err } return Value{value: value, meta: meta}, nil } -func validateScalarValue(value []byte) error { - _, err := validateValue(value) - return err +func validateScalarValue(meta Metadata, value []byte) error { + size, err := validateValue(meta, value, 0) + if err != nil { + return err + } + if size != len(value) { + return fmt.Errorf("invalid variant value: trailing bytes") + } + return nil } -func validateValue(value []byte) (int, error) { +func validateValue(meta Metadata, value []byte, depth int) (int, error) { if len(value) == 0 { return 0, errors.New("invalid variant value: empty") } + if depth > maxValidationDepth { + return 0, fmt.Errorf("invalid variant value: maximum nesting depth exceeded") + } switch basicTypeFromHeader(value[0]) { case BasicShortString: @@ -533,9 +543,9 @@ func validateValue(value []byte) (int, error) { } return want, nil case BasicObject: - return validateObjectValue(value) + return validateObjectValue(meta, value, depth) case BasicArray: - return validateArrayValue(value) + return validateArrayValue(meta, value, depth) case BasicPrimitive: return validatePrimitiveValue(value) default: @@ -586,7 +596,7 @@ func validatePrimitiveValue(value []byte) (int, error) { return want, nil } -func validateArrayValue(value []byte) (int, error) { +func validateArrayValue(meta Metadata, value []byte, depth int) (int, error) { typeInfo := value[0] >> basicTypeBits offsetSize := uint8(typeInfo&0b11) + 1 isLarge := ((typeInfo >> 2) & 0x1) != 0 @@ -633,7 +643,7 @@ func validateArrayValue(value []byte) (int, error) { for i := 0; i < len(offsets)-1; i++ { start := dataStart + uint64(offsets[i]) end := dataStart + uint64(offsets[i+1]) - childSize, err := validateValue(value[int(start):int(end)]) + childSize, err := validateValue(meta, value[int(start):int(end)], depth+1) if err != nil { return 0, fmt.Errorf("invalid variant value: array element %d: %w", i, err) } @@ -645,7 +655,7 @@ func validateArrayValue(value []byte) (int, error) { return int(dataStart + uint64(offsets[len(offsets)-1])), nil } -func validateObjectValue(value []byte) (int, error) { +func validateObjectValue(meta Metadata, value []byte, depth int) (int, error) { typeInfo := value[0] >> basicTypeBits offsetSize := uint8(typeInfo&0b11) + 1 idSize := uint8((typeInfo>>2)&0b11) + 1 @@ -670,35 +680,83 @@ func validateObjectValue(value []byte) (int, error) { return 0, fmt.Errorf("invalid variant value: object offset table ends at %d, got %d bytes", dataStart, len(value)) } - offsets := make([]uint32, int(numElements)+1) - for i := range offsets { - pos := offsetStart + uint64(i)*uint64(offsetSize) - offset := readLEU32(value[int(pos) : int(pos)+int(offsetSize)]) - if i == 0 && offset != 0 { - return 0, fmt.Errorf("invalid variant value: object first offset must be zero, got %d", offset) - } - if i > 0 && offset < offsets[i-1] { - return 0, fmt.Errorf("invalid variant value: object offsets are not monotonic") + type childRange struct { + start uint64 + end uint64 + field int + } + + fieldOffsets := make([]uint32, int(numElements)) + children := make([]childRange, 0, int(numElements)) + var previousKey string + for i := range numElements { + idPos := idStart + uint64(i)*uint64(idSize) + id := readLEU32(value[int(idPos) : int(idPos)+int(idSize)]) + key, err := meta.KeyAt(id) + if err != nil { + return 0, fmt.Errorf("invalid variant value: object field %d has invalid field ID %d: %w", i, id, err) } - if dataStart+uint64(offset) > uint64(len(value)) || dataStart+uint64(offset) > math.MaxUint32 { - return 0, fmt.Errorf("invalid variant value: object offset %d is out of range", offset) + if i > 0 && strings.Compare(previousKey, key) >= 0 { + return 0, fmt.Errorf("invalid variant value: object field names are not strictly sorted at field %d", i) } - offsets[i] = offset + previousKey = key + + offsetPos := offsetStart + uint64(i)*uint64(offsetSize) + fieldOffsets[i] = readLEU32(value[int(offsetPos) : int(offsetPos)+int(offsetSize)]) } - for i := 0; i < len(offsets)-1; i++ { - start := dataStart + uint64(offsets[i]) - end := dataStart + uint64(offsets[i+1]) - childSize, err := validateValue(value[int(start):int(end)]) + finalOffsetPos := offsetStart + uint64(numElements)*uint64(offsetSize) + dataSize := readLEU32(value[int(finalOffsetPos) : int(finalOffsetPos)+int(offsetSize)]) + if dataStart+uint64(dataSize) > uint64(len(value)) || dataStart+uint64(dataSize) > math.MaxUint32 { + return 0, fmt.Errorf("invalid variant value: object data ends at %d, got %d bytes", dataStart+uint64(dataSize), len(value)) + } + + for i, offset := range fieldOffsets { + if uint64(offset) > uint64(dataSize) { + return 0, fmt.Errorf("invalid variant value: object field %d offset %d is out of range", i, offset) + } + + start := dataStart + uint64(offset) + childSize, err := validateValue(meta, value[int(start):], depth+1) if err != nil { return 0, fmt.Errorf("invalid variant value: object field %d: %w", i, err) } - if uint64(childSize) != end-start { - return 0, fmt.Errorf("invalid variant value: object field %d has trailing bytes", i) + end := uint64(offset) + uint64(childSize) + if end > uint64(dataSize) { + return 0, fmt.Errorf("invalid variant value: object field %d extends beyond data", i) } + children = append(children, childRange{start: uint64(offset), end: end, field: i}) } - return int(dataStart + uint64(offsets[len(offsets)-1])), nil + slices.SortFunc(children, func(a, b childRange) int { + switch { + case a.start < b.start: + return -1 + case a.start > b.start: + return 1 + default: + return 0 + } + }) + var ( + next uint64 + previousField int + ) + for _, child := range children { + switch { + case child.start < next: + return 0, fmt.Errorf("invalid variant value: object fields %d and %d overlap", previousField, child.field) + case child.start > next: + return 0, fmt.Errorf("invalid variant value: object data has a gap before field %d", child.field) + } + next = child.end + previousField = child.field + } + if next != uint64(dataSize) { + return 0, fmt.Errorf("invalid variant value: object data has trailing bytes") + } + + return int(dataStart + uint64(dataSize)), nil } // New creates a Value by parsing both the metadata and value bytes. diff --git a/parquet/variant/variant_test.go b/parquet/variant/variant_test.go index 11cbfea3..416c112e 100644 --- a/parquet/variant/variant_test.go +++ b/parquet/variant/variant_test.go @@ -214,6 +214,30 @@ func primitiveHeader(p variant.PrimitiveType) uint8 { return (uint8(p) << 2) } +func objectValueLayout(value []byte) (offsetStart, offsetSize, dataStart int) { + typeInfo := value[0] >> 2 + numElements := int(value[1]) + idSize := int((typeInfo>>2)&0b11) + 1 + offsetSize = int(typeInfo&0b11) + 1 + offsetStart = 2 + numElements*idSize + dataStart = offsetStart + (numElements+1)*offsetSize + return +} + +func twoFieldObject(t *testing.T) variant.Value { + t.Helper() + var b variant.Builder + start := b.Offset() + fields := []variant.FieldEntry{b.NextField(start, "a")} + require.NoError(t, b.AppendInt(1)) + fields = append(fields, b.NextField(start, "b")) + require.NoError(t, b.AppendInt(2)) + require.NoError(t, b.FinishObject(start, fields)) + v, err := b.Build() + require.NoError(t, err) + return v +} + func TestNullValue(t *testing.T) { emptyMeta := variant.EmptyMetadataBytes nullChars := []byte{primitiveHeader(variant.PrimitiveNull)} @@ -677,6 +701,130 @@ func TestInvalidCompoundValue(t *testing.T) { } } +func TestValidateObjectPhysicalOffsets(t *testing.T) { + var b variant.Builder + start := b.Offset() + fields := []variant.FieldEntry{b.NextField(start, "c")} + require.NoError(t, b.AppendInt(3)) + fields = append(fields, b.NextField(start, "a")) + require.NoError(t, b.AppendInt(1)) + fields = append(fields, b.NextField(start, "b")) + require.NoError(t, b.AppendInt(2)) + require.NoError(t, b.FinishObject(start, fields)) + v, err := b.Build() + require.NoError(t, err) + + // The fields are sorted by key, but their values were appended as c, a, b. + // This produces offsets in key order of 2, 4, 0, which is valid Variant data. + parsed, err := variant.NewWithMetadata(v.Metadata(), v.Bytes()) + require.NoError(t, err) + obj := parsed.Value().(variant.ObjectValue) + for _, tt := range []struct { + key string + value int8 + }{ + {key: "a", value: 1}, + {key: "b", value: 2}, + {key: "c", value: 3}, + } { + field, err := obj.ValueByKey(tt.key) + require.NoError(t, err) + assert.Equal(t, tt.value, field.Value.Value()) + } +} + +func TestValidateObjectMetadataAndRanges(t *testing.T) { + base := twoFieldObject(t) + offsetStart, offsetSize, dataStart := objectValueLayout(base.Bytes()) + + t.Run("invalid field ID", func(t *testing.T) { + value := append([]byte(nil), base.Bytes()...) + value[2] = 0xff + _, err := variant.NewWithMetadata(base.Metadata(), value) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid field ID") + }) + + t.Run("unsorted field names", func(t *testing.T) { + value := append([]byte(nil), base.Bytes()...) + value[2], value[3] = value[3], value[2] + _, err := variant.NewWithMetadata(base.Metadata(), value) + require.Error(t, err) + assert.Contains(t, err.Error(), "not strictly sorted") + }) + + t.Run("duplicate field names", func(t *testing.T) { + value := append([]byte(nil), base.Bytes()...) + value[3] = value[2] + _, err := variant.NewWithMetadata(base.Metadata(), value) + require.Error(t, err) + assert.Contains(t, err.Error(), "not strictly sorted") + }) + + t.Run("overlapping ranges", func(t *testing.T) { + value := append([]byte(nil), base.Bytes()...) + value[offsetStart+offsetSize] = 0 + _, err := variant.NewWithMetadata(base.Metadata(), value) + require.Error(t, err) + assert.Contains(t, err.Error(), "overlap") + }) + + t.Run("gap between ranges", func(t *testing.T) { + var b variant.Builder + start := b.Offset() + fields := []variant.FieldEntry{b.NextField(start, "a")} + require.NoError(t, b.AppendInt(1)) + fields = append(fields, b.NextField(start, "b")) + require.NoError(t, b.AppendInt(2)) + // Leave an otherwise valid value in the physical data region without + // assigning it to a field. + require.NoError(t, b.AppendInt(3)) + require.NoError(t, b.FinishObject(start, fields)) + v, err := b.Build() + require.NoError(t, err) + + value := append([]byte(nil), v.Bytes()...) + offsetStart, offsetSize, _ := objectValueLayout(value) + value[offsetStart+offsetSize] = 4 + _, err = variant.NewWithMetadata(v.Metadata(), value) + require.Error(t, err) + assert.Contains(t, err.Error(), "gap") + }) + + t.Run("offset into another value", func(t *testing.T) { + value := append([]byte(nil), base.Bytes()...) + value[offsetStart+offsetSize] = 1 + _, err := variant.NewWithMetadata(base.Metadata(), value) + require.Error(t, err) + assert.Contains(t, err.Error(), "overlap") + }) + + t.Run("top-level trailing bytes", func(t *testing.T) { + value := append(append([]byte(nil), base.Bytes()...), 0) + _, err := variant.NewWithMetadata(base.Metadata(), value) + require.Error(t, err) + assert.Contains(t, err.Error(), "trailing bytes") + }) + + assert.Greater(t, dataStart, offsetStart) +} + +func TestValidateNestedDepth(t *testing.T) { + var nested any = int64(1) + for range 300 { + nested = []any{nested} + } + + var b variant.Builder + require.NoError(t, b.Append(nested)) + v, err := b.Build() + require.NoError(t, err) + + _, err = variant.NewWithMetadata(v.Metadata(), v.Bytes()) + require.Error(t, err) + assert.Contains(t, err.Error(), "maximum nesting depth") +} + func TestInvalidObjectAccess(t *testing.T) { v := loadVariant(t, "object_primitive") obj := v.Value().(variant.ObjectValue) @@ -700,17 +848,9 @@ func TestInvalidObjectAccess(t *testing.T) { // Set field ID to an invalid value corruptBytes[idPosition] = 0xFF - corrupt, err := variant.NewWithMetadata(v.Metadata(), corruptBytes) - require.NoError(t, err) - - corruptObj := corrupt.Value().(variant.ObjectValue) - _, err = corruptObj.FieldAt(0) - require.Error(t, err) - assert.Contains(t, err.Error(), "fieldID") - - _, err = corruptObj.ValueByKey("int_field") + _, err := variant.NewWithMetadata(v.Metadata(), corruptBytes) require.Error(t, err) - assert.Contains(t, err.Error(), "fieldID") + assert.Contains(t, err.Error(), "invalid field ID") }) } From 582ece63014c6f575003505d1ebbc1d9eec9cdda Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Thu, 6 Aug 2026 23:22:14 +0200 Subject: [PATCH 3/3] fix(parquet/variant): validate deeply nested values iteratively --- parquet/variant/variant.go | 315 ++++++++++++++++++++++---------- parquet/variant/variant_test.go | 5 +- 2 files changed, 220 insertions(+), 100 deletions(-) diff --git a/parquet/variant/variant.go b/parquet/variant/variant.go index 0ccb331f..e7e1ff9c 100644 --- a/parquet/variant/variant.go +++ b/parquet/variant/variant.go @@ -135,7 +135,6 @@ const ( supportedVersion = 1 maxShortStringSize = 0x3F metadataMaxSizeLimit = 128 * 1024 * 1024 // 128MB - maxValidationDepth = 256 ) var ( @@ -509,15 +508,15 @@ func NewWithMetadata(meta Metadata, value []byte) (Value, error) { if len(value) == 0 { return Value{}, errors.New("invalid variant value: empty") } - if err := validateScalarValue(meta, value); err != nil { + if err := validateValueBytes(meta, value); err != nil { return Value{}, err } return Value{value: value, meta: meta}, nil } -func validateScalarValue(meta Metadata, value []byte) error { - size, err := validateValue(meta, value, 0) +func validateValueBytes(meta Metadata, value []byte) error { + size, err := validateValue(meta, value) if err != nil { return err } @@ -527,32 +526,6 @@ func validateScalarValue(meta Metadata, value []byte) error { return nil } -func validateValue(meta Metadata, value []byte, depth int) (int, error) { - if len(value) == 0 { - return 0, errors.New("invalid variant value: empty") - } - if depth > maxValidationDepth { - return 0, fmt.Errorf("invalid variant value: maximum nesting depth exceeded") - } - - switch basicTypeFromHeader(value[0]) { - case BasicShortString: - want := 1 + int(value[0]>>basicTypeBits) - if len(value) < want { - return 0, fmt.Errorf("invalid variant value: short string requires %d bytes, got %d", want, len(value)) - } - return want, nil - case BasicObject: - return validateObjectValue(meta, value, depth) - case BasicArray: - return validateArrayValue(meta, value, depth) - case BasicPrimitive: - return validatePrimitiveValue(value) - default: - return 0, fmt.Errorf("invalid variant value: unknown basic type %d", basicTypeFromHeader(value[0])) - } -} - func validatePrimitiveValue(value []byte) (int, error) { primitiveType := primitiveTypeFromHeader(value[0]) want := 0 @@ -596,7 +569,190 @@ func validatePrimitiveValue(value []byte) (int, error) { return want, nil } -func validateArrayValue(meta Metadata, value []byte, depth int) (int, error) { +type validationChild struct { + value []byte + index int + start uint64 + expectedSize uint64 +} + +type validationRange struct { + start uint64 + end uint64 + field int +} + +type validationFrame struct { + value []byte + kind BasicType + size int + dataSize uint64 + children []validationChild + ranges []validationRange + nextChild int + pending validationChild + initialized bool + compound bool +} + +// validateValue walks compound values with an explicit stack so valid values +// are not limited by the Go call stack or an implementation-defined nesting +// depth. +func validateValue(meta Metadata, value []byte) (int, error) { + stack := []validationFrame{{value: value}} + var ( + resultSize int + resultErr error + hasResult bool + ) + + for len(stack) > 0 { + frame := &stack[len(stack)-1] + if hasResult { + child := frame.pending + hasResult = false + + if resultErr != nil { + switch frame.kind { + case BasicArray: + return 0, fmt.Errorf("invalid variant value: array element %d: %w", child.index, resultErr) + case BasicObject: + return 0, fmt.Errorf("invalid variant value: object field %d: %w", child.index, resultErr) + default: + return 0, resultErr + } + } + + switch frame.kind { + case BasicArray: + if uint64(resultSize) != child.expectedSize { + return 0, fmt.Errorf("invalid variant value: array element %d has trailing bytes", child.index) + } + case BasicObject: + end := child.start + uint64(resultSize) + if end > frame.dataSize { + return 0, fmt.Errorf("invalid variant value: object field %d extends beyond data", child.index) + } + frame.ranges = append(frame.ranges, validationRange{ + start: child.start, + end: end, + field: child.index, + }) + } + continue + } + + if !frame.initialized { + frame.initialized = true + if err := prepareValidationFrame(meta, frame); err != nil { + stack = stack[:len(stack)-1] + if len(stack) == 0 { + return 0, err + } + resultErr = err + hasResult = true + continue + } + } + + if frame.compound { + if frame.nextChild < len(frame.children) { + child := frame.children[frame.nextChild] + frame.nextChild++ + frame.pending = child + stack = append(stack, validationFrame{value: child.value}) + continue + } + + if err := finishValidationFrame(frame); err != nil { + stack = stack[:len(stack)-1] + if len(stack) == 0 { + return 0, err + } + resultErr = err + hasResult = true + continue + } + } + + resultSize = frame.size + stack = stack[:len(stack)-1] + if len(stack) == 0 { + return resultSize, nil + } + hasResult = true + } + + return 0, errors.New("invalid variant value: validation stack exhausted") +} + +func finishValidationFrame(frame *validationFrame) error { + if frame.kind != BasicObject { + return nil + } + + slices.SortFunc(frame.ranges, func(a, b validationRange) int { + switch { + case a.start < b.start: + return -1 + case a.start > b.start: + return 1 + default: + return 0 + } + }) + + var ( + next uint64 + previousField int + ) + for _, child := range frame.ranges { + switch { + case child.start < next: + return fmt.Errorf("invalid variant value: object fields %d and %d overlap", previousField, child.field) + case child.start > next: + return fmt.Errorf("invalid variant value: object data has a gap before field %d", child.field) + } + next = child.end + previousField = child.field + } + if next != frame.dataSize { + return fmt.Errorf("invalid variant value: object data has trailing bytes") + } + return nil +} + +func prepareValidationFrame(meta Metadata, frame *validationFrame) error { + if len(frame.value) == 0 { + return errors.New("invalid variant value: empty") + } + + frame.kind = basicTypeFromHeader(frame.value[0]) + switch frame.kind { + case BasicShortString: + want := 1 + int(frame.value[0]>>basicTypeBits) + if len(frame.value) < want { + return fmt.Errorf("invalid variant value: short string requires %d bytes, got %d", want, len(frame.value)) + } + frame.size = want + case BasicObject: + frame.compound = true + return prepareObjectValidationFrame(meta, frame) + case BasicArray: + frame.compound = true + return prepareArrayValidationFrame(frame) + case BasicPrimitive: + var err error + frame.size, err = validatePrimitiveValue(frame.value) + return err + default: + return fmt.Errorf("invalid variant value: unknown basic type %d", frame.kind) + } + return nil +} + +func prepareArrayValidationFrame(frame *validationFrame) error { + value := frame.value typeInfo := value[0] >> basicTypeBits offsetSize := uint8(typeInfo&0b11) + 1 isLarge := ((typeInfo >> 2) & 0x1) != 0 @@ -607,13 +763,13 @@ func validateArrayValue(meta Metadata, value []byte, depth int) (int, error) { ) if isLarge { if len(value) < 5 { - return 0, fmt.Errorf("invalid variant value: array size requires 5 bytes, got %d", len(value)) + return fmt.Errorf("invalid variant value: array size requires 5 bytes, got %d", len(value)) } numElements = readLEU32(value[1:5]) offsetStart = 5 } else { if len(value) < 2 { - return 0, fmt.Errorf("invalid variant value: array size requires 2 bytes, got %d", len(value)) + return fmt.Errorf("invalid variant value: array size requires 2 bytes, got %d", len(value)) } numElements = uint32(value[1]) offsetStart = 2 @@ -621,7 +777,7 @@ func validateArrayValue(meta Metadata, value []byte, depth int) (int, error) { dataStart := offsetStart + (uint64(numElements)+1)*uint64(offsetSize) if dataStart > uint64(len(value)) || dataStart > math.MaxUint32 { - return 0, fmt.Errorf("invalid variant value: array offset table ends at %d, got %d bytes", dataStart, len(value)) + return fmt.Errorf("invalid variant value: array offset table ends at %d, got %d bytes", dataStart, len(value)) } offsets := make([]uint32, int(numElements)+1) @@ -629,33 +785,34 @@ func validateArrayValue(meta Metadata, value []byte, depth int) (int, error) { pos := offsetStart + uint64(i)*uint64(offsetSize) offset := readLEU32(value[int(pos) : int(pos)+int(offsetSize)]) if i == 0 && offset != 0 { - return 0, fmt.Errorf("invalid variant value: array first offset must be zero, got %d", offset) + return fmt.Errorf("invalid variant value: array first offset must be zero, got %d", offset) } if i > 0 && offset < offsets[i-1] { - return 0, fmt.Errorf("invalid variant value: array offsets are not monotonic") + return fmt.Errorf("invalid variant value: array offsets are not monotonic") } if dataStart+uint64(offset) > uint64(len(value)) || dataStart+uint64(offset) > math.MaxUint32 { - return 0, fmt.Errorf("invalid variant value: array offset %d is out of range", offset) + return fmt.Errorf("invalid variant value: array offset %d is out of range", offset) } offsets[i] = offset } + frame.children = make([]validationChild, 0, len(offsets)-1) for i := 0; i < len(offsets)-1; i++ { start := dataStart + uint64(offsets[i]) end := dataStart + uint64(offsets[i+1]) - childSize, err := validateValue(meta, value[int(start):int(end)], depth+1) - if err != nil { - return 0, fmt.Errorf("invalid variant value: array element %d: %w", i, err) - } - if uint64(childSize) != end-start { - return 0, fmt.Errorf("invalid variant value: array element %d has trailing bytes", i) - } + frame.children = append(frame.children, validationChild{ + value: value[int(start):int(end)], + index: i, + expectedSize: end - start, + }) } - return int(dataStart + uint64(offsets[len(offsets)-1])), nil + frame.size = int(dataStart + uint64(offsets[len(offsets)-1])) + return nil } -func validateObjectValue(meta Metadata, value []byte, depth int) (int, error) { +func prepareObjectValidationFrame(meta Metadata, frame *validationFrame) error { + value := frame.value typeInfo := value[0] >> basicTypeBits offsetSize := uint8(typeInfo&0b11) + 1 idSize := uint8((typeInfo>>2)&0b11) + 1 @@ -669,7 +826,7 @@ func validateObjectValue(meta Metadata, value []byte, depth int) (int, error) { elementSize = 4 } if uint64(len(value)) < 1+elementSize { - return 0, fmt.Errorf("invalid variant value: object size requires %d bytes, got %d", 1+elementSize, len(value)) + return fmt.Errorf("invalid variant value: object size requires %d bytes, got %d", 1+elementSize, len(value)) } numElements = readLEU32(value[1 : 1+elementSize]) @@ -677,27 +834,20 @@ func validateObjectValue(meta Metadata, value []byte, depth int) (int, error) { offsetStart := idStart + uint64(numElements)*uint64(idSize) dataStart := offsetStart + (uint64(numElements)+1)*uint64(offsetSize) if dataStart > uint64(len(value)) || dataStart > math.MaxUint32 { - return 0, fmt.Errorf("invalid variant value: object offset table ends at %d, got %d bytes", dataStart, len(value)) - } - - type childRange struct { - start uint64 - end uint64 - field int + return fmt.Errorf("invalid variant value: object offset table ends at %d, got %d bytes", dataStart, len(value)) } fieldOffsets := make([]uint32, int(numElements)) - children := make([]childRange, 0, int(numElements)) var previousKey string for i := range numElements { idPos := idStart + uint64(i)*uint64(idSize) id := readLEU32(value[int(idPos) : int(idPos)+int(idSize)]) key, err := meta.KeyAt(id) if err != nil { - return 0, fmt.Errorf("invalid variant value: object field %d has invalid field ID %d: %w", i, id, err) + return fmt.Errorf("invalid variant value: object field %d has invalid field ID %d: %w", i, id, err) } if i > 0 && strings.Compare(previousKey, key) >= 0 { - return 0, fmt.Errorf("invalid variant value: object field names are not strictly sorted at field %d", i) + return fmt.Errorf("invalid variant value: object field names are not strictly sorted at field %d", i) } previousKey = key @@ -708,55 +858,26 @@ func validateObjectValue(meta Metadata, value []byte, depth int) (int, error) { finalOffsetPos := offsetStart + uint64(numElements)*uint64(offsetSize) dataSize := readLEU32(value[int(finalOffsetPos) : int(finalOffsetPos)+int(offsetSize)]) if dataStart+uint64(dataSize) > uint64(len(value)) || dataStart+uint64(dataSize) > math.MaxUint32 { - return 0, fmt.Errorf("invalid variant value: object data ends at %d, got %d bytes", dataStart+uint64(dataSize), len(value)) + return fmt.Errorf("invalid variant value: object data ends at %d, got %d bytes", dataStart+uint64(dataSize), len(value)) } + frame.children = make([]validationChild, 0, len(fieldOffsets)) for i, offset := range fieldOffsets { if uint64(offset) > uint64(dataSize) { - return 0, fmt.Errorf("invalid variant value: object field %d offset %d is out of range", i, offset) + return fmt.Errorf("invalid variant value: object field %d offset %d is out of range", i, offset) } start := dataStart + uint64(offset) - childSize, err := validateValue(meta, value[int(start):], depth+1) - if err != nil { - return 0, fmt.Errorf("invalid variant value: object field %d: %w", i, err) - } - end := uint64(offset) + uint64(childSize) - if end > uint64(dataSize) { - return 0, fmt.Errorf("invalid variant value: object field %d extends beyond data", i) - } - children = append(children, childRange{start: uint64(offset), end: end, field: i}) - } - - slices.SortFunc(children, func(a, b childRange) int { - switch { - case a.start < b.start: - return -1 - case a.start > b.start: - return 1 - default: - return 0 - } - }) - var ( - next uint64 - previousField int - ) - for _, child := range children { - switch { - case child.start < next: - return 0, fmt.Errorf("invalid variant value: object fields %d and %d overlap", previousField, child.field) - case child.start > next: - return 0, fmt.Errorf("invalid variant value: object data has a gap before field %d", child.field) - } - next = child.end - previousField = child.field - } - if next != uint64(dataSize) { - return 0, fmt.Errorf("invalid variant value: object data has trailing bytes") + frame.children = append(frame.children, validationChild{ + value: value[int(start):], + index: i, + start: uint64(offset), + }) } - return int(dataStart + uint64(dataSize)), nil + frame.dataSize = uint64(dataSize) + frame.size = int(dataStart + uint64(dataSize)) + return nil } // New creates a Value by parsing both the metadata and value bytes. diff --git a/parquet/variant/variant_test.go b/parquet/variant/variant_test.go index 416c112e..0d3134fe 100644 --- a/parquet/variant/variant_test.go +++ b/parquet/variant/variant_test.go @@ -809,7 +809,7 @@ func TestValidateObjectMetadataAndRanges(t *testing.T) { assert.Greater(t, dataStart, offsetStart) } -func TestValidateNestedDepth(t *testing.T) { +func TestValidateDeeplyNestedValue(t *testing.T) { var nested any = int64(1) for range 300 { nested = []any{nested} @@ -821,8 +821,7 @@ func TestValidateNestedDepth(t *testing.T) { require.NoError(t, err) _, err = variant.NewWithMetadata(v.Metadata(), v.Bytes()) - require.Error(t, err) - assert.Contains(t, err.Error(), "maximum nesting depth") + require.NoError(t, err) } func TestInvalidObjectAccess(t *testing.T) {