Skip to content

Commit

Permalink
feat: Add column.type for better compatibility.
Browse files Browse the repository at this point in the history
  • Loading branch information
jheer committed Aug 13, 2024
1 parent 5fd1e1c commit 198f034
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 11 deletions.
10 changes: 10 additions & 0 deletions src/batch.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,16 @@
import { float64 } from './array-types.js';
import { decodeBit, decodeUtf8, divide, readInt32, readInt64AsNum, toNumber } from './util.js';

/**
* Check if the input is a batch that supports direct access to
* binary data in the form of typed arrays.
* @param {Batch<any>?} batch The data batch to check.
* @returns {boolean} True if a direct batch, false otherwise.
*/
export function isDirectBatch(batch) {
return batch instanceof DirectBatch;
}

/**
* Column values from a single record batch.
* A column may contain multiple batches.
Expand Down
26 changes: 17 additions & 9 deletions src/column.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import { DirectBatch } from './batch.js';
import { isDirectBatch } from './batch.js';

/**
* Build up a column from batches.
*/
export function columnBuilder() {
export function columnBuilder(type) {
let data = [];
return {
add(batch) { data.push(batch); return this; },
clear: () => data = [],
done: () => new Column(data)
done: () => new Column(type, data)
};
}

Expand All @@ -23,20 +23,30 @@ export function columnBuilder() {
export class Column {
/**
* Create a new column instance.
* @param {import('./types.js').DataType} type The data type.
* @param {import('./batch.js').Batch<T>[]} data The value batches.
*/
constructor(data) {
constructor(type, data) {
/**
* The column data type.
* @type {import('./types.js').DataType}
* @readonly
*/
this.type = type;
/**
* The column length.
* @type {number}
* @readonly
*/
this.length = data.reduce((m, c) => m + c.length, 0);
/**
* The count of null values in the column.
* @type {number}
* @readonly
*/
this.nullCount = data.reduce((m, c) => m + c.nullCount, 0);
/**
* An array of column data batches.
* @type {readonly import('./batch.js').Batch<T>[]}
* @readonly
*/
Expand All @@ -56,6 +66,8 @@ export class Column {
}

/**
* Index offsets for data batches.
* Used to map a column row index to a batch-specific index.
* @type {Int32Array}
* @readonly
*/
Expand Down Expand Up @@ -115,7 +127,7 @@ export class Column {
*/
toArray() {
const { length, nullCount, data } = this;
const copy = !nullCount && isDirect(data);
const copy = !nullCount && isDirectBatch(data[0]);
const n = data.length;

if (copy && n === 1) {
Expand Down Expand Up @@ -151,10 +163,6 @@ function *batchedIterator(data) {
}
}

function isDirect(data) {
return data.length && data[0] instanceof DirectBatch;
}

function copyArray(array, data) {
for (let i = 0, offset = 0; i < data.length; ++i) {
const { values } = data[i];
Expand Down
4 changes: 2 additions & 2 deletions src/table-from-ipc.js
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ export function createTable(data, options = {}) {
if (isDelta) {
throw new Error('Delta update can not be first dictionary batch.');
}
dicts.set(id, columnBuilder().add(batch));
dicts.set(id, columnBuilder(type).add(batch));
} else {
const dict = dicts.get(id);
if (!isDelta) dict.clear();
Expand All @@ -72,7 +72,7 @@ export function createTable(data, options = {}) {
dicts.forEach((value, key) => dictionaryMap.set(key, value.done()));

// decode column fields
const cols = fields.map(() => columnBuilder());
const cols = fields.map(f => columnBuilder(f.type));
for (const batch of records) {
const ctx = context(batch);
fields.forEach((f, i) => cols[i].add(visit(f.type, ctx)));
Expand Down

0 comments on commit 198f034

Please sign in to comment.