Skip to content

Commit

Permalink
Added tests for loading of simple lists.
Browse files Browse the repository at this point in the history
  • Loading branch information
LTLA committed May 1, 2024
1 parent 00c8291 commit 8240cab
Show file tree
Hide file tree
Showing 5 changed files with 191 additions and 75 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/run-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
- name: Set up Node
uses: actions/setup-node@v3
with:
node-version: 16
node-version: 20

- name: Restore the node modules
uses: actions/cache@v3
Expand Down
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"type": "module",
"main": "src/index.js",
"scripts": {
"test": "node --experimental-vm-modules --experimental-wasm-threads node_modules/jest/bin/jest.js --testTimeout=1000000 --runInBand --detectOpenHandles --forceExit",
"test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testTimeout=1000000 --runInBand --detectOpenHandles --forceExit",
"jsdoc": "npx jsdoc -r src README.md -d docs/built -c docs/jsdoc.config.json"
},
"repository": "github:kanaverse/bakana-takane",
Expand All @@ -28,7 +28,7 @@
},
"devDependencies": {
"docdash": "^2.0.1",
"jest": "^27.5.1",
"jest": "^29.7.0",
"jsdoc": "^4.0.1",
"wasmarrays.js": "^0.1.0"
}
Expand Down
154 changes: 82 additions & 72 deletions src/readers/list.js
Original file line number Diff line number Diff line change
@@ -1,22 +1,24 @@
import * as scran from "scran.js";
import * as utils from "./utils.js";

export async function readSimpleList(list_path, navigator) {
const list_meta = await navigator.fetchObjectMetadata(list_path);

let list_format = "hdf5";
if ("format" in list_meta) {
list_format = list_meta.format;
if ("format" in list_meta.simple_list) {
list_format = list_meta.simple_list.format;
}

if (list_format == "json.gz") {
const contents = await navigator.get(list_path + "/list.json.gz");
const stream = await new Response(contents.buffer).body.pipeThrough(new DecompressionStream('gzip'));
const contents = await navigator.get(list_path + "/list_contents.json.gz");
const res = new Response(contents.buffer);
const stream = await res.body.pipeThrough(new DecompressionStream('gzip'));
const unpacked = await new Response(stream).text();
const parsed = JSON.parse(unpacked);
return parseJsonList(parsed);

} else if (list_format == "hdf5") {
const contents = await navigator.get(list_path + "/list.h5");
const contents = await navigator.get(list_path + "/list_contents.h5");
const realized = scran.realizeFile(contents);
let loaded;
try {
Expand Down Expand Up @@ -67,8 +69,8 @@ function parseJsonList(obj) {
}
});

} else if (obj.type = "number") {
const output = obj.values.map(i => {
} else if (obj.type == "number") {
const output = obj.values.map(x => {
if (typeof x != "string") {
return x;
} else if (x == "Inf") {
Expand Down Expand Up @@ -116,7 +118,7 @@ function parseHdf5List(handle) {
return output;

} else {
const children = dhandle.children.keys();
const children = Object.keys(dhandle.children);
const output = new Array(children.length);
for (const i of children) {
output[Number(i)] = parseHdf5List(dhandle.open(i, { load: true }));
Expand All @@ -127,83 +129,91 @@ function parseHdf5List(handle) {
} else if (objtype == "nothing") {
return null;

} else if (objtype == "factor") {
const levels = handle.open("levels", { load: true }).values;
const ihandle = handle.open("values", { load: true });
const codes = ihandle.values;

const output = new Array(codes.length);
if ("missing-value-placeholder" in ihandle.attributes) {
const placeholder = ihandle.readAttribute("missing-value-placeholder").values[0];
for (const [i, x] of codes.entries()) {
if (i == placeholder) {
output[i] = null;
} else {
output[i] = levels[x];
}
}
} else {
for (const [i, x] of codes.entries()) {
output[i] = levels[x];
} else if (objtype == "vector") {
const vectype = handle.readAttribute("uzuki_type").values[0];

if (vectype == "string") {
const vhandle = handle.open("data", { load: true });
if (vhandle.attributes.indexOf("missing-value-placeholder") >= 0) {
const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0];
return vhandle.values.map(x => {
if (x == placeholder) {
return null;
} else {
return x;
}
});
} else {
return vhandle.values;
}
}

return output;

} else if (objtype = "string") {
const vhandle = handle.open("values", { load: true });
if ("missing-value-placeholder" in vhandle.attributes) {
const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0];
return vhandle.values.map(x => {
if (x == placeholder) {
return null;
} else {
return x;
} else if (vectype == "boolean") {
const vhandle = handle.open("data", { load: true });
const values = vhandle.values;

const output = new Array(values.length);
if (vhandle.attributes.indexOf("missing-value-placeholder") >= 0) {
const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0];
for (const [i, x] of values.entries()) {
if (x == placeholder) {
output[i] = null;
} else {
output[i] = (x != 0);
}
}
});
} else {
return vhandle.values;
}

} else if (objtype == "boolean") {
const vhandle = handle.open("values", { load: true });
const values = vhandle.values;

const output = new Array(values.length);
if ("missing-value-placeholder" in vhandle.attributes) {
const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0];
for (const [i, x] of values.entries()) {
if (i == placeholder) {
output[i] = null;
} else {
} else {
for (const [i, x] of values.entries()) {
output[i] = (x != 0);
}
}
} else {
for (const [i, x] of values.entries()) {
output[i] = (x != 0);
}
}

return output;
return output;

} else if (vectype == "integer" || vectype == "number") {
const vhandle = handle.open("data", { load: true });
let output = vhandle.values;

} else if (objtype = "integer" || objtype == "number") {
const vhandle = handle.open("values", { load: true });
let output = vhandle.values;
if (vhandle.attributes.indexOf("missing-value-placeholder") >= 0) {
const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0];
output = utils.substitutePlaceholder(output, placeholder);
} else if (vectype == "number") {
if (!(output instanceof Float64Array) && !(output instanceof Float32Array)) {
output = new Float64Array(output);
}
}

if ("missing-value-placeholder" in vhandle.attributes) {
const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0];
output = utils.substitutePlaceholder(output, placeholder);
} else if (objtype == "number") {
if (!(output instanceof Float64Array) && !(output instanceof Float32Array)) {
output = new Float64Array(output);
return output;

} else if (vectype == "factor") {
const levels = handle.open("levels", { load: true }).values;
const ihandle = handle.open("data", { load: true });
const codes = ihandle.values;

const output = new Array(codes.length);
if (ihandle.attributes.indexOf("missing-value-placeholder") >= 0) {
const placeholder = ihandle.readAttribute("missing-value-placeholder").values[0];
for (const [i, x] of codes.entries()) {
if (x == placeholder) {
output[i] = null;
} else {
output[i] = levels[x];
}
}
} else {
for (const [i, x] of codes.entries()) {
output[i] = levels[x];
}
}
}

return output;
return output;

} else {
console.warn("HDF5 simple list containing a vector of type '" + vectype + "' is not yet supported");
return null;
}

} else {
console.warn("JSON simple list containing type '" + objtype + "' is not yet supported");
console.warn("HDF5 simple list containing type '" + objtype + "' is not yet supported");
return null;
}
}
55 changes: 55 additions & 0 deletions tests/readers/list.setup.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
library(alabaster.base)
PATH <- "objects"
dir.create(PATH, showWarnings=FALSE)
library(S4Vectors)

{
df <- list(
strings = "A",
integers = 1:3,
numbers = 4:6/2,
booleans = FALSE,
factors = factor("Z"),
nothing = NULL
)

path <- file.path(PATH, "list-basic")
unlink(path, recursive=TRUE)
dir.create(path)

saveObject(df, file.path(path, "js"))
saveObject(df, file.path(path, "h5"), list.format='hdf5')
}

{
df <- list(
strings = c("a", NA_character_),
integers = c(1L, NA, 2L),
numbers = c(3.5, NA, 4.5),
booleans = c(NA, TRUE),
factors = factor(c("Z", NA, "A"))
)

path <- file.path(PATH, "list-missing")
unlink(path, recursive=TRUE)
dir.create(path)

saveObject(df, file.path(path, "js"))
saveObject(df, file.path(path, "h5"), list.format='hdf5')
}

{
df <- list(
named = list(A = 1L, B = 2L),
unnamed = list("X", "Y", "Z"),
other = DataFrame(B = 2)
)

path <- file.path(PATH, "list-nested")
unlink(path, recursive=TRUE)
dir.create(path)

saveObject(df, file.path(path, "js"))
saveObject(df, file.path(path, "h5"), list.format='hdf5')
}

51 changes: 51 additions & 0 deletions tests/readers/list.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import * as list from "../../src/readers/list.js";
import { localNavigator } from "../utils.js";
import * as path from "path";
import * as scran from "scran.js";

beforeAll(async () => { await scran.initialize({ localFile: true }) });
afterAll(async () => { await scran.terminate() });

const PATH = "objects";

test("basic list loading works as expected", async () => {
const basic_list = await list.readSimpleList(path.join(PATH, "list-basic", "js"), localNavigator);
expect(Object.keys(basic_list)).toEqual(["strings", "integers", "numbers", "booleans", "factors", "nothing" ]);

expect(basic_list["strings"]).toEqual(["A"]);
expect(basic_list["integers"]).toEqual(new Int32Array([1,2,3]));
expect(basic_list["numbers"]).toEqual(new Float64Array([2,2.5,3]));
expect(basic_list["booleans"]).toEqual([false]);
expect(basic_list["factors"]).toEqual(["Z"]);
expect(basic_list["nothing"]).toBeNull();

const basic_list_h5 = await list.readSimpleList(path.join(PATH, "list-basic", "h5"), localNavigator);
expect(basic_list_h5).toEqual(basic_list);
})

test("list loading works with missing values", async () => {
const missing_list = await list.readSimpleList(path.join(PATH, "list-missing", "js"), localNavigator);
expect(Object.keys(missing_list)).toEqual(["strings", "integers", "numbers", "booleans", "factors" ]);

expect(missing_list["strings"]).toEqual(["a", null]);
expect(missing_list["integers"]).toEqual([1,null,2]);
expect(missing_list["numbers"]).toEqual([3.5,null,4.5]);
expect(missing_list["booleans"]).toEqual([null, true]);
expect(missing_list["factors"]).toEqual(["Z", null, "A"]);

const missing_list_h5 = await list.readSimpleList(path.join(PATH, "list-missing", "h5"), localNavigator);
expect(missing_list_h5).toEqual(missing_list);
})

test("list loading works with nesting", async () => {
const nested_list = await list.readSimpleList(path.join(PATH, "list-nested", "js"), localNavigator);
expect(Object.keys(nested_list)).toEqual(["named", "unnamed", "other"]);

expect(nested_list["named"]).toEqual({ A: new Int32Array([1]), B: new Int32Array([2]) });
expect(nested_list["unnamed"]).toEqual([["X"], ["Y"], ["Z"]]);
expect(nested_list["other"]).toBeNull();

const nested_list_h5 = await list.readSimpleList(path.join(PATH, "list-nested", "h5"), localNavigator);
expect(nested_list_h5).toEqual(nested_list);
})

0 comments on commit 8240cab

Please sign in to comment.