Add more gold integration tests.

Sources/Arrow/Array/Array.swift

@@ -49,6 +49,16 @@ public protocol ArrowArrayOfData {
 extension ArrowArrayFixedSizeBinary: ArrowArrayOfData where ItemType == Data {}
 extension ArrowArrayVariable: ArrowArrayOfData where ItemType == Data {}
 
+public protocol ArrowArrayOfInt8 {
+  subscript(index: Int) -> Int8? { get }
+}
+extension ArrowArrayFixed: ArrowArrayOfInt8 where ItemType == Int8 {}
+
+public protocol ArrowArrayOfInt32 {
+  subscript(index: Int) -> Int32? { get }
+}
+extension ArrowArrayFixed: ArrowArrayOfInt32 where ItemType == Int32 {}
+
 /// An Arrow array of booleans using the three-valued logical model (true / false / null).
 public struct ArrowArrayBoolean: ArrowArrayProtocol {
   public typealias ItemType = Bool
@@ -382,12 +392,12 @@ public struct AnyArrowListArray: ArrowArrayProtocol {
   public let length: Int
   public var nullCount: Int { _base.nullCount }
 
-  init<Element, OffsetsBuffer>(
+  public init<Element, OffsetsBuffer>(
     _ list: ArrowListArray<Element, OffsetsBuffer>
   )
   where
     OffsetsBuffer: FixedWidthBufferProtocol<Int32>,
-    Element: ArrowArrayProtocol
+    Element: AnyArrowArrayProtocol
   {
     self._base = list
     self.offset = list.offset

Sources/ArrowIPC/ArrowReader.swift

@@ -71,13 +71,9 @@ struct FixedWidthBufferIPC<Element>: FixedWidthBufferProtocol, ArrowBufferIPC
 where
   Element: Numeric, Element: BitwiseCopyable
 {
-
   typealias ElementType = Element
-
   let buffer: FileDataBuffer
-  var length: Int {
-    buffer.range.count
-  }
+  var length: Int { buffer.range.count }
 
   subscript(index: Int) -> Element {
     buffer.data.withUnsafeBytes { rawBuffer in
@@ -93,12 +89,8 @@ struct VariableLengthBufferIPC<Element: VariableLength>:
   VariableLengthBufferProtocol, ArrowBufferIPC
 {
   typealias ElementType = Element
-
   let buffer: FileDataBuffer
-
-  var length: Int {
-    buffer.range.count
-  }
+  var length: Int { buffer.range.count }
 
   func loadVariable(
     at startIndex: Int,
@@ -285,6 +277,22 @@ public struct ArrowReader {
         return makeFixedArray(
           length: length, elementType: Double.self,
           nullBuffer: nullBuffer, buffer: buffer1)
+      case .int8:
+        return makeFixedArray(
+          length: length, elementType: Int8.self,
+          nullBuffer: nullBuffer, buffer: buffer1)
+      case .int16:
+        return makeFixedArray(
+          length: length, elementType: Int16.self,
+          nullBuffer: nullBuffer, buffer: buffer1)
+      case .int32:
+        return makeFixedArray(
+          length: length, elementType: Int32.self,
+          nullBuffer: nullBuffer, buffer: buffer1)
+      case .int64:
+        return makeFixedArray(
+          length: length, elementType: Int64.self,
+          nullBuffer: nullBuffer, buffer: buffer1)
       default:
         throw ArrowError.notImplemented
       }
@@ -321,11 +329,21 @@ public struct ArrowReader {
           nodeIndex: &nodeIndex,
           bufferIndex: &bufferIndex
         )
-
         let buffer1 = try nextBuffer(
           message: rbMessage, index: &bufferIndex, offset: offset, data: data)
-        let offsetsBuffer = FixedWidthBufferIPC<Int32>(buffer: buffer1)
-
+        var offsetsBuffer = FixedWidthBufferIPC<Int32>(buffer: buffer1)
+
+        // TODO: This is a hack for the special-case where buffer length 0 means all-zero offset.
+        // Can follow the null buffer example.
+        if offsetsBuffer.length != length + 1 {
+          let offsetCount = length + 1
+          let byteCount = offsetCount * MemoryLayout<Int32>.stride
+          let fileDataBuffer = FileDataBuffer(
+            data: Data(count: byteCount),  // Zero-initialized
+            range: 0..<byteCount
+          )
+          offsetsBuffer = FixedWidthBufferIPC<Int32>(buffer: fileDataBuffer)
+        }
         return makeListArray(
           length: length,
           nullBuffer: nullBuffer,
@@ -411,9 +429,7 @@ public struct ArrowReader {
       offsetsBuffer: offsetsBuffer,
       values: values
     )
-    // FIXME: Need to fix list types.
-    fatalError()
-    //    return AnyArrowListArray(list)
+    return AnyArrowListArray(list)
   }
 
   private func loadSchema(_ schema: FSchema) throws(ArrowError) -> ArrowSchema {

Tests/ArrowIPCTests/ArrowReaderTests.swift

@@ -20,7 +20,7 @@ import Testing
 struct ArrowReaderTests {
 
   @Test func boolFile() throws {
-    let url = try loadArrowResource(name: "testdata_bool")
+    let url = try loadTestResource(name: "testdata_bool")
     let arrowReader = try ArrowReader(url: url)
     let (arrowSchema, recordBatches) = try arrowReader.read()
     for recordBatch in recordBatches {
@@ -30,7 +30,7 @@ struct ArrowReaderTests {
 
   @Test func doubleFile() throws {
 
-    let url = try loadArrowResource(name: "testdata_double")
+    let url = try loadTestResource(name: "testdata_double")
     let arrowReader = try ArrowReader(url: url)
     let (arrowSchema, recordBatches) = try arrowReader.read()
 
@@ -68,7 +68,7 @@ struct ArrowReaderTests {
   }
 
   @Test func structFile() throws {
-    let url = try loadArrowResource(name: "testdata_struct")
+    let url = try loadTestResource(name: "testdata_struct")
     let arrowReader = try ArrowReader(url: url)
     let (arrowSchema, recordBatches) = try arrowReader.read()
     for recordBatch in recordBatches {

Tests/ArrowIPCTests/Gold/ArrowTestingFormat.swift → Tests/ArrowIPCTests/Gold/ArrowGold.swift

@@ -14,8 +14,8 @@
 
 import Foundation
 
-/// The JSON structure used to validate Arrow test files.
-struct ArrowTestingFormat: Codable {
+/// The JSON file structure used to validate gold-standard  Arrow test files.
+struct ArrowGold: Codable {
   let schema: Schema
   let batches: [Batch]
   let dictionaries: [Dictionary]?
@@ -46,6 +46,7 @@ struct ArrowTestingFormat: Codable {
   struct FieldType: Codable {
     let name: String
     let byteWidth: Int?
+    let bitWidth: Int?
     let isSigned: Bool?
     let precision: String?
     let scale: Int?
@@ -63,7 +64,7 @@ struct ArrowTestingFormat: Codable {
     let count: Int
     let validity: [Int]?
     let offset: [Int]?
-    let data: [String]?
+    let data: [DataValue]?
     let children: [Column]?
 
     enum CodingKeys: String, CodingKey {
@@ -82,3 +83,32 @@ struct ArrowTestingFormat: Codable {
     case bool(Bool)
   }
 }
+
+/// Arrow gold files data values have variable types.
+enum DataValue: Codable {
+  case string(String)
+  case int(Int)
+  case double(Double)
+  case null
+
+  init(from decoder: Decoder) throws {
+    let container = try decoder.singleValueContainer()
+
+    if container.decodeNil() {
+      self = .null
+    } else if let intValue = try? container.decode(Int.self) {
+      self = .int(intValue)
+    } else if let doubleValue = try? container.decode(Double.self) {
+      self = .double(doubleValue)
+    } else if let stringValue = try? container.decode(String.self) {
+      self = .string(stringValue)
+    } else {
+      throw DecodingError.typeMismatch(
+        DataValue.self,
+        DecodingError.Context(
+          codingPath: decoder.codingPath,
+          debugDescription: "Cannot decode DataValue")
+      )
+    }
+  }
+}