From cc6684904c74312285b5f4dd0500e64c1d251a9d Mon Sep 17 00:00:00 2001 From: Will Temperley Date: Sat, 22 Nov 2025 11:48:41 +0800 Subject: [PATCH 1/2] Add more Arrow gold test cases. --- Sources/Arrow/Array/Array.swift | 4 +- Sources/ArrowIPC/ArrowReader.swift | 20 ++++- Tests/ArrowIPCTests/ArrowReaderTests.swift | 6 +- ...rowTestingFormat.swift => ArrowGold.swift} | 35 ++++++++- .../ArrowIPCTests/Gold/ArrowTestingIPC.swift | 75 ++++++++++--------- Tests/ArrowIPCTests/TestSupport.swift | 12 +-- 6 files changed, 102 insertions(+), 50 deletions(-) rename Tests/ArrowIPCTests/Gold/{ArrowTestingFormat.swift => ArrowGold.swift} (62%) diff --git a/Sources/Arrow/Array/Array.swift b/Sources/Arrow/Array/Array.swift index 2d6b3f9..fa89f9d 100644 --- a/Sources/Arrow/Array/Array.swift +++ b/Sources/Arrow/Array/Array.swift @@ -382,12 +382,12 @@ public struct AnyArrowListArray: ArrowArrayProtocol { public let length: Int public var nullCount: Int { _base.nullCount } - init( + public init( _ list: ArrowListArray ) where OffsetsBuffer: FixedWidthBufferProtocol, - Element: ArrowArrayProtocol + Element: AnyArrowArrayProtocol { self._base = list self.offset = list.offset diff --git a/Sources/ArrowIPC/ArrowReader.swift b/Sources/ArrowIPC/ArrowReader.swift index 5653f4a..1593baa 100644 --- a/Sources/ArrowIPC/ArrowReader.swift +++ b/Sources/ArrowIPC/ArrowReader.swift @@ -285,6 +285,22 @@ public struct ArrowReader { return makeFixedArray( length: length, elementType: Double.self, nullBuffer: nullBuffer, buffer: buffer1) + case .int8: + return makeFixedArray( + length: length, elementType: Int8.self, + nullBuffer: nullBuffer, buffer: buffer1) + case .int16: + return makeFixedArray( + length: length, elementType: Int16.self, + nullBuffer: nullBuffer, buffer: buffer1) + case .int32: + return makeFixedArray( + length: length, elementType: Int32.self, + nullBuffer: nullBuffer, buffer: buffer1) + case .int64: + return makeFixedArray( + length: length, elementType: Int64.self, + nullBuffer: nullBuffer, buffer: buffer1) default: throw ArrowError.notImplemented } @@ -412,8 +428,8 @@ public struct ArrowReader { values: values ) // FIXME: Need to fix list types. - fatalError() - // return AnyArrowListArray(list) +// fatalError() + return AnyArrowListArray(list) } private func loadSchema(_ schema: FSchema) throws(ArrowError) -> ArrowSchema { diff --git a/Tests/ArrowIPCTests/ArrowReaderTests.swift b/Tests/ArrowIPCTests/ArrowReaderTests.swift index a2aac59..36c4fc7 100644 --- a/Tests/ArrowIPCTests/ArrowReaderTests.swift +++ b/Tests/ArrowIPCTests/ArrowReaderTests.swift @@ -20,7 +20,7 @@ import Testing struct ArrowReaderTests { @Test func boolFile() throws { - let url = try loadArrowResource(name: "testdata_bool") + let url = try loadTestResource(name: "testdata_bool") let arrowReader = try ArrowReader(url: url) let (arrowSchema, recordBatches) = try arrowReader.read() for recordBatch in recordBatches { @@ -30,7 +30,7 @@ struct ArrowReaderTests { @Test func doubleFile() throws { - let url = try loadArrowResource(name: "testdata_double") + let url = try loadTestResource(name: "testdata_double") let arrowReader = try ArrowReader(url: url) let (arrowSchema, recordBatches) = try arrowReader.read() @@ -68,7 +68,7 @@ struct ArrowReaderTests { } @Test func structFile() throws { - let url = try loadArrowResource(name: "testdata_struct") + let url = try loadTestResource(name: "testdata_struct") let arrowReader = try ArrowReader(url: url) let (arrowSchema, recordBatches) = try arrowReader.read() for recordBatch in recordBatches { diff --git a/Tests/ArrowIPCTests/Gold/ArrowTestingFormat.swift b/Tests/ArrowIPCTests/Gold/ArrowGold.swift similarity index 62% rename from Tests/ArrowIPCTests/Gold/ArrowTestingFormat.swift rename to Tests/ArrowIPCTests/Gold/ArrowGold.swift index 63f0ac9..b6a3559 100644 --- a/Tests/ArrowIPCTests/Gold/ArrowTestingFormat.swift +++ b/Tests/ArrowIPCTests/Gold/ArrowGold.swift @@ -14,8 +14,8 @@ import Foundation -/// The JSON structure used to validate Arrow test files. -struct ArrowTestingFormat: Codable { +/// The JSON file structure used to validate gold-standard Arrow test files. +struct ArrowGold: Codable { let schema: Schema let batches: [Batch] let dictionaries: [Dictionary]? @@ -46,6 +46,7 @@ struct ArrowTestingFormat: Codable { struct FieldType: Codable { let name: String let byteWidth: Int? + let bitWidth: Int? let isSigned: Bool? let precision: String? let scale: Int? @@ -63,7 +64,7 @@ struct ArrowTestingFormat: Codable { let count: Int let validity: [Int]? let offset: [Int]? - let data: [String]? + let data: [DataValue]? let children: [Column]? enum CodingKeys: String, CodingKey { @@ -82,3 +83,31 @@ struct ArrowTestingFormat: Codable { case bool(Bool) } } + +/// Arrow gold files data values have variable types. +enum DataValue: Codable { + case string(String) + case int(Int) + case double(Double) + case null + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + + if container.decodeNil() { + self = .null + } else if let intValue = try? container.decode(Int.self) { + self = .int(intValue) + } else if let doubleValue = try? container.decode(Double.self) { + self = .double(doubleValue) + } else if let stringValue = try? container.decode(String.self) { + self = .string(stringValue) + } else { + throw DecodingError.typeMismatch( + DataValue.self, + DecodingError.Context(codingPath: decoder.codingPath, + debugDescription: "Cannot decode DataValue") + ) + } + } +} diff --git a/Tests/ArrowIPCTests/Gold/ArrowTestingIPC.swift b/Tests/ArrowIPCTests/Gold/ArrowTestingIPC.swift index 0b7148d..bc67940 100644 --- a/Tests/ArrowIPCTests/Gold/ArrowTestingIPC.swift +++ b/Tests/ArrowIPCTests/Gold/ArrowTestingIPC.swift @@ -19,37 +19,35 @@ import Testing @testable import ArrowIPC struct ArrowTestingIPC { - - @Test func generatedBinary() throws { - - guard - let resourceURL = Bundle.module.url( - forResource: "Resources/integration/cpp-21.0.0/generated_binary.json", - withExtension: "lz4" - ) - else { - throw ArrowError.invalid("Unable to locate generated_binary.json") - } - + + static let testCases: [String] = [ + "generated_binary", + // "generated_binary_view", + "generated_binary_zerolength", + "generated_binary_no_batches", +// "generated_custom_metadata" + ] + + + @Test(arguments: testCases) + func gold(name: String) throws { + + let resourceURL = try loadTestResource( + name: name, + withExtension: "json.lz4", + subdirectory: "integration/cpp-21.0.0" + ) let lz4Data = try Data(contentsOf: resourceURL) let lz4 = try LZ4(parsing: lz4Data) - let testCase = try JSONDecoder().decode( - ArrowTestingFormat.self, from: lz4.data) - - // try printTestJSON(testCase) - - guard - let testFile = Bundle.module.url( - forResource: "Resources/integration/cpp-21.0.0/generated_binary", - withExtension: "arrow_file" - ) - else { - throw ArrowError.invalid("Unable to locate arrow file.") - } - + let testCase = try JSONDecoder().decode(ArrowGold.self, from: lz4.data) + let testFile = try loadTestResource( + name: name, + withExtension: "arrow_file", + subdirectory: "integration/cpp-21.0.0" + ) let arrowReader = try ArrowReader(url: testFile) let (arrowSchema, recordBatches) = try arrowReader.read() - + #expect(testCase.batches.count == recordBatches.count) for (testBatch, recordBatch) in zip(testCase.batches, recordBatches) { @@ -76,10 +74,10 @@ struct ArrowTestingIPC { } try testFixedWidthBinary(actual: actual, expected: expectedColumn) case .binary: - try testVariable( + try testVariableLength( actual: arrowArray, expected: expectedColumn, type: arrowField.type) case .utf8: - try testVariable( + try testVariableLength( actual: arrowArray, expected: expectedColumn, type: arrowField.type) default: print(arrowField.type) @@ -91,14 +89,17 @@ struct ArrowTestingIPC { func testFixedWidthBinary( actual: ArrowArrayOfData, - expected: ArrowTestingFormat.Column, + expected: ArrowGold.Column, ) throws { guard let validity = expected.validity, let dataValues = expected.data else { throw ArrowError.invalid("Test column is incomplete.") } + for (i, isNull) in validity.enumerated() { - let hex = dataValues[i] + guard case .string(let hex) = dataValues[i] else { + throw ArrowError.invalid("Data values are not all strings.") + } guard let data = Data(hex: hex) else { Issue.record("Failed to decode data from hex: \(hex)") return @@ -111,9 +112,9 @@ struct ArrowTestingIPC { } } - func testVariable( + func testVariableLength( actual: AnyArrowArrayProtocol, - expected: ArrowTestingFormat.Column, + expected: ArrowGold.Column, type: ArrowType ) throws { guard let expectedValidity = expected.validity, @@ -136,7 +137,9 @@ struct ArrowTestingIPC { return } for i in 0.. URL { +func loadTestResource( + name: String, withExtension ext: String = "arrow", subdirectory: String = "" +) throws(ArrowError) -> URL { if let resource = Bundle.module.url( forResource: name, - withExtension: "arrow", - subdirectory: "Resources" + withExtension: ext, + subdirectory: "Resources/\(subdirectory)" ) { return resource } else { - throw .runtimeError("Couldn't find \(name).arrow in the test resources.") + throw .runtimeError("Couldn't find \(name).\(ext) in the test resources.") } } From 4e1ac787de87859f6b7b6fa4b268a31569936528 Mon Sep 17 00:00:00 2001 From: Will Temperley Date: Sat, 22 Nov 2025 14:25:57 +0800 Subject: [PATCH 2/2] Add list array tests to gold tests. --- Sources/Arrow/Array/Array.swift | 10 ++ Sources/ArrowIPC/ArrowReader.swift | 32 ++--- Tests/ArrowIPCTests/Gold/ArrowGold.swift | 47 +++--- .../ArrowIPCTests/Gold/ArrowTestingIPC.swift | 134 ++++++++++++++++-- 4 files changed, 175 insertions(+), 48 deletions(-) diff --git a/Sources/Arrow/Array/Array.swift b/Sources/Arrow/Array/Array.swift index fa89f9d..12026ea 100644 --- a/Sources/Arrow/Array/Array.swift +++ b/Sources/Arrow/Array/Array.swift @@ -49,6 +49,16 @@ public protocol ArrowArrayOfData { extension ArrowArrayFixedSizeBinary: ArrowArrayOfData where ItemType == Data {} extension ArrowArrayVariable: ArrowArrayOfData where ItemType == Data {} +public protocol ArrowArrayOfInt8 { + subscript(index: Int) -> Int8? { get } +} +extension ArrowArrayFixed: ArrowArrayOfInt8 where ItemType == Int8 {} + +public protocol ArrowArrayOfInt32 { + subscript(index: Int) -> Int32? { get } +} +extension ArrowArrayFixed: ArrowArrayOfInt32 where ItemType == Int32 {} + /// An Arrow array of booleans using the three-valued logical model (true / false / null). public struct ArrowArrayBoolean: ArrowArrayProtocol { public typealias ItemType = Bool diff --git a/Sources/ArrowIPC/ArrowReader.swift b/Sources/ArrowIPC/ArrowReader.swift index 1593baa..1b6f643 100644 --- a/Sources/ArrowIPC/ArrowReader.swift +++ b/Sources/ArrowIPC/ArrowReader.swift @@ -71,13 +71,9 @@ struct FixedWidthBufferIPC: FixedWidthBufferProtocol, ArrowBufferIPC where Element: Numeric, Element: BitwiseCopyable { - typealias ElementType = Element - let buffer: FileDataBuffer - var length: Int { - buffer.range.count - } + var length: Int { buffer.range.count } subscript(index: Int) -> Element { buffer.data.withUnsafeBytes { rawBuffer in @@ -93,12 +89,8 @@ struct VariableLengthBufferIPC: VariableLengthBufferProtocol, ArrowBufferIPC { typealias ElementType = Element - let buffer: FileDataBuffer - - var length: Int { - buffer.range.count - } + var length: Int { buffer.range.count } func loadVariable( at startIndex: Int, @@ -337,11 +329,21 @@ public struct ArrowReader { nodeIndex: &nodeIndex, bufferIndex: &bufferIndex ) - let buffer1 = try nextBuffer( message: rbMessage, index: &bufferIndex, offset: offset, data: data) - let offsetsBuffer = FixedWidthBufferIPC(buffer: buffer1) - + var offsetsBuffer = FixedWidthBufferIPC(buffer: buffer1) + + // TODO: This is a hack for the special-case where buffer length 0 means all-zero offset. + // Can follow the null buffer example. + if offsetsBuffer.length != length + 1 { + let offsetCount = length + 1 + let byteCount = offsetCount * MemoryLayout.stride + let fileDataBuffer = FileDataBuffer( + data: Data(count: byteCount), // Zero-initialized + range: 0..(buffer: fileDataBuffer) + } return makeListArray( length: length, nullBuffer: nullBuffer, @@ -427,9 +429,7 @@ public struct ArrowReader { offsetsBuffer: offsetsBuffer, values: values ) - // FIXME: Need to fix list types. -// fatalError() - return AnyArrowListArray(list) + return AnyArrowListArray(list) } private func loadSchema(_ schema: FSchema) throws(ArrowError) -> ArrowSchema { diff --git a/Tests/ArrowIPCTests/Gold/ArrowGold.swift b/Tests/ArrowIPCTests/Gold/ArrowGold.swift index b6a3559..6e952dc 100644 --- a/Tests/ArrowIPCTests/Gold/ArrowGold.swift +++ b/Tests/ArrowIPCTests/Gold/ArrowGold.swift @@ -86,28 +86,29 @@ struct ArrowGold: Codable { /// Arrow gold files data values have variable types. enum DataValue: Codable { - case string(String) - case int(Int) - case double(Double) - case null - - init(from decoder: Decoder) throws { - let container = try decoder.singleValueContainer() - - if container.decodeNil() { - self = .null - } else if let intValue = try? container.decode(Int.self) { - self = .int(intValue) - } else if let doubleValue = try? container.decode(Double.self) { - self = .double(doubleValue) - } else if let stringValue = try? container.decode(String.self) { - self = .string(stringValue) - } else { - throw DecodingError.typeMismatch( - DataValue.self, - DecodingError.Context(codingPath: decoder.codingPath, - debugDescription: "Cannot decode DataValue") - ) - } + case string(String) + case int(Int) + case double(Double) + case null + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + + if container.decodeNil() { + self = .null + } else if let intValue = try? container.decode(Int.self) { + self = .int(intValue) + } else if let doubleValue = try? container.decode(Double.self) { + self = .double(doubleValue) + } else if let stringValue = try? container.decode(String.self) { + self = .string(stringValue) + } else { + throw DecodingError.typeMismatch( + DataValue.self, + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Cannot decode DataValue") + ) } + } } diff --git a/Tests/ArrowIPCTests/Gold/ArrowTestingIPC.swift b/Tests/ArrowIPCTests/Gold/ArrowTestingIPC.swift index bc67940..0a72b87 100644 --- a/Tests/ArrowIPCTests/Gold/ArrowTestingIPC.swift +++ b/Tests/ArrowIPCTests/Gold/ArrowTestingIPC.swift @@ -19,19 +19,18 @@ import Testing @testable import ArrowIPC struct ArrowTestingIPC { - + static let testCases: [String] = [ "generated_binary", - // "generated_binary_view", + // "generated_binary_view", "generated_binary_zerolength", "generated_binary_no_batches", -// "generated_custom_metadata" + "generated_custom_metadata", ] - @Test(arguments: testCases) func gold(name: String) throws { - + let resourceURL = try loadTestResource( name: name, withExtension: "json.lz4", @@ -47,7 +46,7 @@ struct ArrowTestingIPC { ) let arrowReader = try ArrowReader(url: testFile) let (arrowSchema, recordBatches) = try arrowReader.read() - + #expect(testCase.batches.count == recordBatches.count) for (testBatch, recordBatch) in zip(testCase.batches, recordBatches) { @@ -79,9 +78,18 @@ struct ArrowTestingIPC { case .utf8: try testVariableLength( actual: arrowArray, expected: expectedColumn, type: arrowField.type) + case .int8: + try testFixedWidth( + actual: arrowArray, expected: expectedColumn, as: Int8.self) + case .int32: + try testFixedWidth( + actual: arrowArray, expected: expectedColumn, as: Int32.self) + case .list(_): + try validateListArray(actual: arrowArray, expected: expectedColumn) + break default: - print(arrowField.type) - throw ArrowError.notImplemented + throw ArrowError.invalid( + "Unsupported arrow field type: \(arrowField.type)") } } } @@ -95,7 +103,7 @@ struct ArrowTestingIPC { else { throw ArrowError.invalid("Test column is incomplete.") } - + for (i, isNull) in validity.enumerated() { guard case .string(let hex) = dataValues[i] else { throw ArrowError.invalid("Data values are not all strings.") @@ -112,6 +120,114 @@ struct ArrowTestingIPC { } } + func testFixedWidth( + actual: AnyArrowArrayProtocol, + expected: ArrowGold.Column, + as type: T.Type + ) throws where T: BinaryInteger { + guard let expectedValidity = expected.validity, + let expectedValues = expected.data + else { + throw ArrowError.invalid("Test column is incomplete.") + } + + guard let array = actual as? any ArrowArrayProtocol, + array.length == expectedValidity.count + else { + Issue.record("Array type mismatch") + return + } + + for (i, isNull) in expectedValidity.enumerated() { + guard case .int(let val) = expectedValues[i] else { + throw ArrowError.invalid("Expected integer value") + } + + let expected = try T(throwingOnOverflow: val) + + if isNull == 0 { + #expect(array[i] == nil) + } else { + #expect(array[i] as? T == expected) + } + } + } + + func validateListArray( + actual: AnyArrowArrayProtocol, + expected: ArrowGold.Column + ) throws { + guard let expectedValidity = expected.validity, + let expectedOffsets = expected.offset + else { + throw ArrowError.invalid("Test column is incomplete.") + } + + // Validate the offsets buffer + actual.buffers[1].withUnsafeBytes { ptr in + let offsets = ptr.bindMemory(to: Int32.self) + #expect(offsets.count == expectedOffsets.count) + for (i, expectedOffset) in expectedOffsets.enumerated() { + #expect(offsets[i] == expectedOffset) + } + } + + guard let listArray = actual as? AnyArrowListArray else { + Issue.record("Unexpected array type") + return + } + + guard let child = expected.children?.first else { + throw ArrowError.invalid("List array missing child column") + } + + // Validate each list entry + for (i, isNull) in expectedValidity.enumerated() { + if isNull == 0 { + #expect(listArray[i] == nil) + } else { + guard let actualChildSlice = listArray[i] else { + Issue.record("Expected non-null list at index \(i)") + continue + } + + // Get expected range from offsets + let childStartOffset = Int(expectedOffsets[i]) + let childEndOffset = Int(expectedOffsets[i + 1]) + let expectedLength = childEndOffset - childStartOffset + + #expect(actualChildSlice.length == expectedLength) + + // Validate each element in this list + for j in 0..