Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import PackageDescription
let package = Package(
name: "Arrow",
platforms: [
.macOS(.v26)
.macOS(.v26), .iOS(.v26), .watchOS(.v26), .tvOS(.v26), .visionOS(.v26),
],
products: [
.library(name: "Arrow", targets: ["Arrow"]),
Expand Down
12 changes: 6 additions & 6 deletions Sources/Arrow/Array/Builder.swift
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public class ArrayBuilderBoolean: AnyArrayBuilder {
}

/// A builder for Arrow arrays holding fixed-width types.
public class ArrayBuilderFixedWidth<T: Numeric & BitwiseCopyable>:
public class ArrayBuilderNumeric<T: Numeric & BitwiseCopyable>:
AnyArrayBuilder
{

Expand Down Expand Up @@ -234,7 +234,7 @@ typealias ArrayBuilderBinary = ArrayBuilderVariableLength<Data, Int32>
/// A builder for Arrow arrays holding `Date`s with a resolution of one day.
public struct ArrayBuilderDate32: AnyArrayBuilder {
public typealias ArrayType = ArrowArrayDate32
let builder: ArrayBuilderFixedWidth<Date32> = .init()
let builder: ArrayBuilderNumeric<Date32> = .init()

public init() {}

Expand All @@ -259,7 +259,7 @@ public struct ArrayBuilderDate32: AnyArrayBuilder {
/// A builder for Arrow arrays holding `Date`s with a resolution of one day.
public struct ArrayBuilderDate64: AnyArrayBuilder {
public typealias ArrayType = ArrowArrayDate64
let builder: ArrayBuilderFixedWidth<Date64> = .init()
let builder: ArrayBuilderNumeric<Date64> = .init()

public init() {}

Expand All @@ -282,13 +282,13 @@ public struct ArrayBuilderDate64: AnyArrayBuilder {
}

/// A builder for Arrow arrays holding Time32 values.
public typealias ArrayBuilderTime32 = ArrayBuilderFixedWidth<Time32>
public typealias ArrayBuilderTime32 = ArrayBuilderNumeric<Time32>

/// A builder for Arrow arrays holding Time64 values.
public typealias ArrayBuilderTime64 = ArrayBuilderFixedWidth<Time64>
public typealias ArrayBuilderTime64 = ArrayBuilderNumeric<Time64>

/// A builder for Arrow arrays holding Timestamp values.
public typealias ArrayBuilderTimestamp = ArrayBuilderFixedWidth<Timestamp>
public typealias ArrayBuilderTimestamp = ArrayBuilderNumeric<Timestamp>

public class ArrayBuilderList<T: AnyArrayBuilder> {

Expand Down
68 changes: 68 additions & 0 deletions Sources/Arrow/Array/DictionaryArray.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// Copyright 2026 The Columnar Swift Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/// A shared container for the dictionary values.
public class DictionaryValues: @unchecked Sendable {
var currentArray: AnyArrowArrayProtocol

init(_ array: AnyArrowArrayProtocol) {
self.currentArray = array
}
}

///// An Arrow dictionary array.
public struct ArrowDictionaryArray<
IndexType: FixedWidthInteger & BitwiseCopyable
>: ArrowArrayProtocol {
public let offset: Int
public let length: Int
public var bufferSizes: [Int] { keys.bufferSizes }
public var buffers: [ArrowBufferProtocol] { keys.buffers }
public var nullCount: Int { keys.nullCount }

public let keys: ArrowArrayNumeric<IndexType>
public let values: DictionaryValues

public init(
offset: Int = 0,
length: Int,
keys: ArrowArrayNumeric<IndexType>,
values: AnyArrowArrayProtocol
) {
self.offset = offset
self.length = length
self.keys = keys
self.values = DictionaryValues(values)
}

public subscript(index: Int) -> Any? {
precondition(index >= 0 && index < length, "Invalid index.")
let offsetIndex = self.offset + index
guard let key = keys[offsetIndex] else {
return nil
}
precondition(
Int(key) < values.currentArray.length, "Key out of bounds for dictionary")
return values.currentArray.any(at: Int(key))
}

public func slice(offset: Int, length: Int) -> Self {
.init(
offset: 0,
length: length,
keys: keys.slice(offset: self.offset + offset, length: length),
values: values.currentArray
)
}
}
33 changes: 15 additions & 18 deletions Sources/Arrow/ArrowField.swift
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ public struct ArrowField: Codable, Sendable {
///
/// If true, the field *may* contain null values.
public var isNullable: Bool
public let orderedDict: Bool
/// A map of key-value pairs containing additional custom meta data.
public var metadata: [String: String]
}
Expand Down Expand Up @@ -60,7 +59,6 @@ extension ArrowField {
self.name = name
self.type = dataType
self.isNullable = isNullable
self.orderedDict = false
self.metadata = metadata
}

Expand All @@ -76,21 +74,20 @@ extension ArrowField {
)
}

/// Create a new `ArrowField` suitable for `ArrowType::Dictionary`.
///
public init(
dictWithName: String,
key: ArrowType,
value: ArrowType,
isNullable: Bool
) {
precondition(
key.isDictionaryKeyType,
"\(key) is not a valid dictionary key"
)
let dataType: ArrowType = .dictionary(key, value)
self = Self(name: dictWithName, dataType: dataType, isNullable: isNullable)
}
// /// Create a new `ArrowField` suitable for `ArrowType::Dictionary`.
// public init(
// dictWithName: String,
// key: ArrowType,
// value: ArrowType,
// isNullable: Bool
// ) {
// precondition(
// key.isDictionaryKeyType,
// "\(key) is not a valid dictionary key"
// )
// let dataType: ArrowType = .dictionary(key, value)
// self = Self(name: dictWithName, dataType: dataType, isNullable: isNullable)
// }

/// Create a new struct `ArrowField`.
///
Expand Down Expand Up @@ -218,7 +215,7 @@ extension ArrowField {
@inlinable
public var dictIsOrdered: Bool {
switch self.type {
case .dictionary: return self.orderedDict
case .dictionary(_, let isOrdered, _, _): return isOrdered
default: return false
}
}
Expand Down
13 changes: 7 additions & 6 deletions Sources/Arrow/ArrowType.swift
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ public indirect enum ArrowType: Codable, Sendable, Equatable {
///
/// This type mostly used to represent low cardinality string
/// arrays or a limited set of primitive types as integers.
case dictionary(ArrowType, ArrowType)
case dictionary(id: Int64, isOrdered: Bool, key: ArrowType, value: ArrowType)
/// Exact 32-bit width decimal value with precision and scale
///
/// * precision is the total number of digits
Expand Down Expand Up @@ -496,8 +496,8 @@ extension ArrowType: CustomStringConvertible {
return "LargeListView(\(elementType))"
case .union(let mode, let fields):
return "Union(\(mode), \(fields) fields)"
case .dictionary(let keyType, let valueType):
return "Dictionary(\(keyType), \(valueType))"
case .dictionary(let id, let isOrdered, let keyType, let valueType):
return "Dictionary(\(id), \(isOrdered), \(keyType), \(valueType))"
case .decimal32(let precision, let scale):
return "Decimal32(\(precision), \(scale))"
case .decimal64(let precision, let scale):
Expand Down Expand Up @@ -627,7 +627,7 @@ extension ArrowType {
@inlinable
public var isNested: Bool {
switch self {
case .dictionary(_, let v):
case .dictionary(_, _, _, let v):
return v.isNested
case .runEndEncoded(_, let v):
return v.type.isNested
Expand Down Expand Up @@ -699,7 +699,8 @@ extension ArrowType {
&& aField.type.equalsDataType(bField.type) && aSorted == bSorted

// Dictionary
case (.dictionary(let aKey, let aValue), .dictionary(let bKey, let bValue)):
case (.dictionary(_, _, let aKey, let aValue), .dictionary(_, _, let bKey, let bValue)):
// Ignoring dictionary id here.
return aKey.equalsDataType(bKey) && aValue.equalsDataType(bValue)

// RunEndEncoded
Expand Down Expand Up @@ -855,7 +856,7 @@ extension ArrowType {
}

// Dictionary
case (.dictionary(let k1, let v1), .dictionary(let k2, let v2)):
case (.dictionary(_, _, let k1, let v1), .dictionary(_, _, let k2, let v2)):
return k1.contains(k2) && v1.contains(v2)

// Base case: equality
Expand Down
45 changes: 45 additions & 0 deletions Sources/ArrowIPC/ArrowReader.swift
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,51 @@ public struct ArrowReader {
let arrowSchema = try Self.loadSchema(schema: schema)
var recordBatches: [RecordBatch] = []

for block in footer.dictionaries {
let (message, bodyOffset) = try data.withParserSpan { input in
try input.seek(toAbsoluteOffset: block.offset)
let marker = try UInt32(parsingLittleEndian: &input)
if marker != continuationMarker {
throw ArrowError(.invalid("Missing continuation marker."))
}
let messageLength = try UInt32(parsingLittleEndian: &input)
let data = try [UInt8](parsing: &input, byteCount: Int(messageLength))
var mbb = ByteBuffer(data: Data(data))
let message: FMessage = getRoot(byteBuffer: &mbb)
let offset = Int64(input.startPosition)
return (message, offset)
}

guard message.headerType == .dictionarybatch else {
throw ArrowError(.invalid("Expected DictionaryBatch message."))
}

guard let dictMessage = message.header(type: FDictionaryBatch.self) else {
throw ArrowError(.invalid("Expected DictionaryBatch as message header"))
}

// 1. Get the Dictionary ID and 'isDelta' flag
let dictId = dictMessage.id
let isDelta = dictMessage.isDelta

// 2. The dictionary data is actually just a RecordBatch with ONE column
// The schema for this internal batch is defined by the dictionary type
// found in the global Schema for this specific ID.
guard let rbMessage = dictMessage.data else {
throw ArrowError(.invalid("DictionaryBatch has no data"))
}

let dictBatch = try Self.loadRecordBatch(
data: self.data,
arrowSchema: arrowSchema,
rbMessage: rbMessage,
offset: bodyOffset
)

// 4. Update the "Box" in your provider
// try dictionaryProvider.update(id: dictId, array: dictionaryArray, isDelta: isDelta)
}

// MARK: Record batch parsing
for block in footer.recordBatches {

Expand Down
10 changes: 8 additions & 2 deletions Sources/ArrowIPC/FlatBuffersTypeAliases.swift
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,22 @@
// limitations under the License.

typealias FFooter = org_apache_arrow_flatbuf_Footer

typealias FMessageHeader = org_apache_arrow_flatbuf_MessageHeader
typealias FMessage = org_apache_arrow_flatbuf_Message

typealias FBlock = org_apache_arrow_flatbuf_Block
typealias FField = org_apache_arrow_flatbuf_Field
typealias FSchema = org_apache_arrow_flatbuf_Schema
typealias FBuffer = org_apache_arrow_flatbuf_Buffer
typealias FFieldNode = org_apache_arrow_flatbuf_FieldNode
typealias FRecordBatch = org_apache_arrow_flatbuf_RecordBatch
typealias FMessageHeader = org_apache_arrow_flatbuf_MessageHeader
typealias FKeyValue = org_apache_arrow_flatbuf_KeyValue

// MARK: Record batches.
typealias FRecordBatch = org_apache_arrow_flatbuf_RecordBatch
typealias FDictionaryBatch = org_apache_arrow_flatbuf_DictionaryBatch
typealias FDictionaryEncoding = org_apache_arrow_flatbuf_DictionaryEncoding

// MARK: Top level type.
typealias FType = org_apache_arrow_flatbuf_Type_

Expand Down
4 changes: 2 additions & 2 deletions Tests/ArrowIPCTests/ArrowTestingGold.swift
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,10 @@ struct ArrowTestingGold {
"generated_union",
]

@Test
// @Test
func readFile() throws {

let name = "generated_nested_large_offsets"
let name = "generated_dictionary"
let (testFile, testCase) = try loadTestCase(
name: name, fileExtension: "arrow_file")
let arrowReader = try ArrowReader(url: testFile)
Expand Down
4 changes: 2 additions & 2 deletions Tests/ArrowTests/ArrayNullBufferTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ struct ArrayNullBufferTests {

@Test func allValidValues() throws {
// Should be able to omit null buffer entirely
let arrayBuilder: ArrayBuilderFixedWidth<Int64> = .init()
let arrayBuilder: ArrayBuilderNumeric<Int64> = .init()
for i in 0..<1000 {
arrayBuilder.append(Int64(i)) // No nulls
}
Expand All @@ -35,7 +35,7 @@ struct ArrayNullBufferTests {
}

@Test func allNullValues() throws {
let arrayBuilder: ArrayBuilderFixedWidth<Int64> = .init()
let arrayBuilder: ArrayBuilderNumeric<Int64> = .init()
for _ in 0..<1000 {
arrayBuilder.appendNull()
}
Expand Down
4 changes: 2 additions & 2 deletions Tests/ArrowTests/BasicArrayTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ struct BasicArrayTests {
}

@Test func uint8Array() throws {
let arrayBuilder: ArrayBuilderFixedWidth<UInt8> = .init()
let arrayBuilder: ArrayBuilderNumeric<UInt8> = .init()
for index: UInt8 in 0..<100 {
arrayBuilder.append(index)
}
Expand Down Expand Up @@ -105,7 +105,7 @@ struct BasicArrayTests {
}

@Test func doubleArray() throws {
let builder: ArrayBuilderFixedWidth<Double> = .init()
let builder: ArrayBuilderNumeric<Double> = .init()
builder.append(14)
builder.appendNull()
builder.append(40.4)
Expand Down
4 changes: 2 additions & 2 deletions Tests/ArrowTests/ChunkedArrayTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ struct ChunkedArrayTests {
: (remaining <= numChunks - chunks.count)
? 1 : Int.random(in: 1...(remaining - (numChunks - chunks.count - 1)))

let builder = ArrayBuilderFixedWidth<Int32>()
let builder = ArrayBuilderNumeric<Int32>()
for i in 0..<chunkSize {
let val = flatArray[offset + i]
if let val {
Expand Down Expand Up @@ -72,7 +72,7 @@ struct ChunkedArrayTests {
var flatArray: [Int32?] = []
var chunks: [any ArrowArrayProtocol<Int32>] = []
for chunkIdx in 0..<10 {
let builder = ArrayBuilderFixedWidth<Int32>()
let builder = ArrayBuilderNumeric<Int32>()
for i in 0..<size {
let value = Int32(chunkIdx * size + i)
flatArray.append(value)
Expand Down
Loading
Loading