Add null counts to arrays. Initial work on ArrowWriter.

README.md

@@ -6,6 +6,104 @@ A Swift implementation of Apache Arrow, the universal columnar format for fast d
 
 This is a **work in progress**. Do not use in production. Progress is fast however, expect a beta in December.
 
+## Array interface
+
+Arrow arrays are backed by a standard memory layout:
+https://arrow.apache.org/docs/format/Columnar.html
+
+In Swift-Arrow, every array conforms to: 
+
+```swift
+public protocol ArrowArrayProtocol {
+  associatedtype ItemType
+  subscript(_ index: Int) -> ItemType? { get }
+  var offset: Int { get }
+  var length: Int { get }
+  func slice(offset: Int, length: Int) -> Self
+  func any(at index: Int) -> Any?
+}
+```
+
+The in-memory contiguous buffers allow constant-time random access.
+
+Every Arrow array supports nullable elements. This is encoded as an optional bit-packed validity buffer aka null array aka bitfield. 
+In psuedocode, bitfield[index] == 0 means null or invalid, and bitfield[index] == 1 means not null or valid.
+Fixed-width types are encoded back-to-back, with placeholder values for nulls. For example the array:
+
+```swift
+let swiftArray: [Int8?] = [1, nil, 2, 3, nil, 4]
+let arrayBuilder: ArrayBuilderFixedWidth<Int8> = .init()
+for value in swiftArray {
+  if let value {
+    arrayBuilder.append(value)
+  } else {
+    arrayBuilder.appendNull()
+  }
+}
+let arrowArray = arrayBuilder.finish()
+for i in 0..<swiftArray.count {
+  #expect(arrowArray[i] == swiftArray[i])
+}
+```
+
+would be backed by a values buffer of `Int8`:
+
+`[1, 0, 2, 3, 0, 4]`
+
+and a bit-packed validity buffer of UInt8:
+`[45]` or `[b00101101]`
+
+Note the validity buffer may be empty if all values are null, or all values are non null.
+
+Arrow Arrays of variable-length types such as `String` have an offsets buffer. For example:
+
+```swift
+let swiftArray: [String?] = ["ab", nil, "c", "", "."]
+let arrayBuilder: ArrayBuilderVariable<String> = .init()
+for value in swiftArray {
+  if let value {
+    arrayBuilder.append(value)
+  } else {
+    arrayBuilder.appendNull()
+  }
+}
+let arrowArray = arrayBuilder.finish()
+#expect(arrowArray[0] == "ab")
+#expect(arrowArray[1] == nil)
+#expect(arrowArray[2] == "c")
+#expect(arrowArray[3] == "")
+#expect(arrowArray[4] == ".")
+```
+
+would have an offsets array of array length + 1 integers:
+`[0, 2, 2, 3, 3, 4]`
+
+This is a lookup into the value array, i.e.:
+
+```swift
+let values: [UInt8] = [97, 98, 99, 46]
+print(values[0..<2]) // [97, 98]
+print(values[2..<2]) // []
+print(values[2..<3]) // [99]
+print(values[3..<4]) // [46]
+```
+
+In practice, buffers can be any contingous storage. In Swift-Arrow, arrays created in memory are usually backed by pointers, whereas arrays loaded from IPC files are backed by memory-mapped `Data` instances.
+
+Arrays can be configured to use different buffer types, by specifying the types as 
+`public struct ArrowArrayVariable<OffsetsBuffer, ValueBuffer>`
+
+this allows the buffer types to be user-specified, e.g.:
+```
+typealias ArrowArrayUtf8 = ArrowArrayVariable<
+  FixedWidthBufferIPC<Int32>,
+  VariableLengthBufferIPC<String>
+>
+``
+
+
+## Relationship to Arrow-Swift
+
 This project is based on Arrow-Swift, the official Swift implementation of Apache Arrow. The decision was made to at least temporarily operate independently of the Apache Software Foundation (ASF). Currently there are no active ASF maintaners with knowledge of Swift, and the only [Apache approved CI for Swift](https://github.com/apache/infrastructure-actions/blob/main/approved_patterns.yml) is [setup-swift which is unmaintained](https://github.com/swift-actions/setup-swift/issues), leading to intermittent CI failures. This has led to delays in much-needed fixes being implemented.
 
 The intention is to continue contributing to the official Apache-Swift repository, however changes can be iterated on more quickly here.

Sources/Arrow/Array/Array.swift

@@ -19,8 +19,10 @@ public protocol ArrowArrayProtocol {
   subscript(_ index: Int) -> ItemType? { get }
   var offset: Int { get }
   var length: Int { get }
+  var nullCount: Int { get }
   func slice(offset: Int, length: Int) -> Self
   func any(at index: Int) -> Any?
+  var bufferSizes: [Int] { get }
 }
 
 // This exists to support type-erased struct arrays.
@@ -35,6 +37,8 @@ public struct ArrowArrayBoolean: ArrowArrayProtocol {
   public typealias ItemType = Bool
   public let offset: Int
   public let length: Int
+  public var bufferSizes: [Int] { [nullBuffer.length, valueBuffer.length] }
+  public var nullCount: Int { nullBuffer.nullCount }
   let nullBuffer: NullBuffer
   let valueBuffer: NullBuffer
 
@@ -75,11 +79,12 @@ where
   ValueBuffer: FixedWidthBufferProtocol,
   ValueBuffer.ElementType: Numeric
 {
-  public typealias ItemType = ValueBuffer.ElementType
 
-  //  public typealias ItemType = Element
+  public typealias ItemType = ValueBuffer.ElementType
   public let offset: Int
   public let length: Int
+  public var bufferSizes: [Int] { [nullBuffer.length, valueBuffer.length] }
+  public var nullCount: Int { nullBuffer.nullCount }
   let nullBuffer: NullBuffer
   let valueBuffer: ValueBuffer
 
@@ -115,16 +120,20 @@ where
 }
 
 /// An Arrow array of variable-length types.
-public struct ArrowArrayVariable<Element, OffsetsBuffer, ValueBuffer>:
+public struct ArrowArrayVariable<OffsetsBuffer, ValueBuffer>:
   ArrowArrayProtocol
 where
-  Element: VariableLength,
   OffsetsBuffer: FixedWidthBufferProtocol<Int32>,
-  ValueBuffer: VariableLengthBufferProtocol<Element>
+  ValueBuffer: VariableLengthBufferProtocol<ValueBuffer.ElementType>,
+  ValueBuffer.ElementType: VariableLength
 {
-  public typealias ItemType = Element
+  public typealias ItemType = ValueBuffer.ElementType
   public let offset: Int
   public let length: Int
+  public var bufferSizes: [Int] {
+    [nullBuffer.length, offsetsBuffer.length, valueBuffer.length]
+  }
+  public var nullCount: Int { nullBuffer.nullCount }
   let nullBuffer: NullBuffer
   let offsetsBuffer: OffsetsBuffer
   let valueBuffer: ValueBuffer
@@ -143,10 +152,8 @@ where
     self.valueBuffer = valueBuffer
   }
 
-  public subscript(index: Int) -> Element? {
-
+  public subscript(index: Int) -> ValueBuffer.ElementType? {
     let offsetIndex = self.offset + index
-
     if !self.nullBuffer.isSet(offsetIndex) {
       return nil
     }
@@ -170,23 +177,18 @@ where
 }
 
 /// An Arrow array of `Date`s with a resolution of 1 day.
-struct ArrowArrayDate32<ValueBuffer>: ArrowArrayProtocol
+public struct ArrowArrayDate32<ValueBuffer>: ArrowArrayProtocol
 where
   ValueBuffer: FixedWidthBufferProtocol<Int32>
 {
-  typealias ItemType = Date
-
+  public typealias ItemType = Date
+  public var bufferSizes: [Int] { array.bufferSizes }
+  public var nullCount: Int { array.nullCount }
+  public var offset: Int { array.offset }
+  public var length: Int { array.length }
   let array: ArrowArrayFixed<ValueBuffer>
 
-  var offset: Int {
-    array.offset
-  }
-
-  var length: Int {
-    array.length
-  }
-
-  subscript(index: Int) -> Date? {
+  public subscript(index: Int) -> Date? {
     precondition(index >= 0 && index < length, "Invalid index.")
     let offsetIndex = self.offset + index
     let days: Int32? = array[offsetIndex]
@@ -197,30 +199,25 @@ where
     }
   }
 
-  func slice(offset: Int, length: Int) -> Self {
+  public func slice(offset: Int, length: Int) -> Self {
     let internalSlice = array.slice(offset: offset, length: length)
     return .init(array: internalSlice)
   }
 }
 
 /// An Arrow array of `Date`s with a resolution of 1 second.
-struct ArrowArrayDate64<ValueBuffer>: ArrowArrayProtocol
+public struct ArrowArrayDate64<ValueBuffer>: ArrowArrayProtocol
 where
   ValueBuffer: FixedWidthBufferProtocol<Date64>
 {
-  typealias ItemType = Date
-
+  public typealias ItemType = Date
+  public var bufferSizes: [Int] { array.bufferSizes }
+  public var nullCount: Int { array.nullCount }
+  public var offset: Int { array.offset }
+  public var length: Int { array.length }
   let array: ArrowArrayFixed<ValueBuffer>
 
-  var offset: Int {
-    array.offset
-  }
-
-  var length: Int {
-    array.length
-  }
-
-  subscript(index: Int) -> Date? {
+  public subscript(index: Int) -> Date? {
     precondition(index >= 0 && index < length, "Invalid index.")
     let offsetIndex = self.offset + index
     let milliseconds: Int64? = array[offsetIndex]
@@ -231,26 +228,44 @@ where
     }
   }
 
-  func slice(offset: Int, length: Int) -> Self {
+  public func slice(offset: Int, length: Int) -> Self {
     let internalSlice = array.slice(offset: offset, length: length)
     return .init(array: internalSlice)
   }
 }
 
-/// An Arrow list array which may be nested arbitrarily.
-struct ArrowListArray<Element>: ArrowArrayProtocol
+/// A strongly-typed Arrow list array which may be nested arbitrarily.
+public struct ArrowListArray<Element, OffsetsBuffer>: ArrowArrayProtocol
 where
+  OffsetsBuffer: FixedWidthBufferProtocol<Int32>,
   Element: ArrowArrayProtocol
 {
-  typealias ItemType = Element
-
-  let offset: Int
-  let length: Int
+  public typealias ItemType = Element
+  public let offset: Int
+  public let length: Int
+  public var bufferSizes: [Int] {
+    [nullBuffer.length, offsetsBuffer.length, values.length]
+  }
+  public var nullCount: Int { nullBuffer.nullCount }
   let nullBuffer: NullBuffer
-  let offsetsBuffer: FixedWidthBuffer<Int32>
+  let offsetsBuffer: OffsetsBuffer
   let values: Element
 
-  subscript(index: Int) -> Element? {
+  public init(
+    offset: Int = 0,
+    length: Int,
+    nullBuffer: NullBuffer,
+    offsetsBuffer: OffsetsBuffer,
+    values: Element
+  ) {
+    self.offset = offset
+    self.length = length
+    self.nullBuffer = nullBuffer
+    self.offsetsBuffer = offsetsBuffer
+    self.values = values
+  }
+
+  public subscript(index: Int) -> Element? {
     precondition(index >= 0 && index < length, "Invalid index.")
     let offsetIndex = self.offset + index
     if !self.nullBuffer.isSet(offsetIndex) {
@@ -263,7 +278,7 @@ where
     return values.slice(offset: Int(startIndex), length: Int(length))
   }
 
-  func slice(offset: Int, length: Int) -> Self {
+  public func slice(offset: Int, length: Int) -> Self {
     .init(
       offset: self.offset + offset,
       length: length,
@@ -274,14 +289,54 @@ where
   }
 }
 
+/// A type-erased wrapper for an Arrow list array.
+public struct AnyArrowListArray: ArrowArrayProtocol {
+
+  public typealias ItemType = any ArrowArrayProtocol
+  public var bufferSizes: [Int] {
+    _base.bufferSizes
+  }
+
+  private let _base: any ArrowArrayProtocol
+  private let _subscriptImpl: (Int) -> (any ArrowArrayProtocol)?
+  private let _sliceImpl: (Int, Int) -> AnyArrowListArray
+
+  public let offset: Int
+  public let length: Int
+  public var nullCount: Int { _base.nullCount }
+
+  public init<Element, OffsetsBuffer>(
+    _ list: ArrowListArray<Element, OffsetsBuffer>
+  )
+  where
+    OffsetsBuffer: FixedWidthBufferProtocol<Int32>,
+    Element: ArrowArrayProtocol
+  {
+    self._base = list
+    self.offset = list.offset
+    self.length = list.length
+    self._subscriptImpl = { list[$0] }
+    self._sliceImpl = { AnyArrowListArray(list.slice(offset: $0, length: $1)) }
+  }
+
+  public subscript(index: Int) -> (any ArrowArrayProtocol)? {
+    _subscriptImpl(index)
+  }
+
+  public func slice(offset: Int, length: Int) -> AnyArrowListArray {
+    _sliceImpl(offset, length)
+  }
+}
+
 /// An Arrow struct array.
 public struct ArrowStructArray: ArrowArrayProtocol {
   public typealias ItemType = [String: Any]
-
-  let nullBuffer: NullBuffer
   public let offset: Int
   public let length: Int
   public let fields: [(name: String, array: any ArrowArrayProtocol)]
+  public var bufferSizes: [Int] { [nullBuffer.length] }
+  public var nullCount: Int { nullBuffer.nullCount }
+  let nullBuffer: NullBuffer
 
   public init(
     offset: Int = 0,