From 3878eebabbc56d9fbb59899e2a3904fcdb0078a5 Mon Sep 17 00:00:00 2001 From: LTLA Date: Wed, 1 May 2024 08:37:12 -0700 Subject: [PATCH] Added tests for loading of simple lists. --- .github/workflows/run-tests.yaml | 4 +- package.json | 4 +- src/readers/list.js | 154 ++++++++++++++++--------------- tests/readers/list.setup.R | 55 +++++++++++ tests/readers/list.test.js | 51 ++++++++++ 5 files changed, 192 insertions(+), 76 deletions(-) create mode 100644 tests/readers/list.setup.R create mode 100644 tests/readers/list.test.js diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 037c755..b24af1e 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -32,7 +32,7 @@ jobs: - name: Build test objects run: | - find tests -name "*.R" -exec R -f {} + + find tests -name "*.R" -exec R -f {} \; - name: Upload test files uses: actions/upload-artifact@v3 @@ -49,7 +49,7 @@ jobs: - name: Set up Node uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 20 - name: Restore the node modules uses: actions/cache@v3 diff --git a/package.json b/package.json index 8f64636..7369492 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "main": "src/index.js", "scripts": { - "test": "node --experimental-vm-modules --experimental-wasm-threads node_modules/jest/bin/jest.js --testTimeout=1000000 --runInBand --detectOpenHandles --forceExit", + "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testTimeout=1000000 --runInBand --detectOpenHandles --forceExit", "jsdoc": "npx jsdoc -r src README.md -d docs/built -c docs/jsdoc.config.json" }, "repository": "github:kanaverse/bakana-takane", @@ -28,7 +28,7 @@ }, "devDependencies": { "docdash": "^2.0.1", - "jest": "^27.5.1", + "jest": "^29.7.0", "jsdoc": "^4.0.1", "wasmarrays.js": "^0.1.0" } diff --git a/src/readers/list.js b/src/readers/list.js index 9daff50..91ddb60 100644 --- a/src/readers/list.js +++ b/src/readers/list.js @@ -1,22 +1,24 @@ import * as scran from "scran.js"; +import * as utils from "./utils.js"; export async function readSimpleList(list_path, navigator) { const list_meta = await navigator.fetchObjectMetadata(list_path); let list_format = "hdf5"; - if ("format" in list_meta) { - list_format = list_meta.format; + if ("format" in list_meta.simple_list) { + list_format = list_meta.simple_list.format; } if (list_format == "json.gz") { - const contents = await navigator.get(list_path + "/list.json.gz"); - const stream = await new Response(contents.buffer).body.pipeThrough(new DecompressionStream('gzip')); + const contents = await navigator.get(list_path + "/list_contents.json.gz"); + const res = new Response(contents.buffer); + const stream = await res.body.pipeThrough(new DecompressionStream('gzip')); const unpacked = await new Response(stream).text(); const parsed = JSON.parse(unpacked); return parseJsonList(parsed); } else if (list_format == "hdf5") { - const contents = await navigator.get(list_path + "/list.h5"); + const contents = await navigator.get(list_path + "/list_contents.h5"); const realized = scran.realizeFile(contents); let loaded; try { @@ -67,8 +69,8 @@ function parseJsonList(obj) { } }); - } else if (obj.type = "number") { - const output = obj.values.map(i => { + } else if (obj.type == "number") { + const output = obj.values.map(x => { if (typeof x != "string") { return x; } else if (x == "Inf") { @@ -116,7 +118,7 @@ function parseHdf5List(handle) { return output; } else { - const children = dhandle.children.keys(); + const children = Object.keys(dhandle.children); const output = new Array(children.length); for (const i of children) { output[Number(i)] = parseHdf5List(dhandle.open(i, { load: true })); @@ -127,83 +129,91 @@ function parseHdf5List(handle) { } else if (objtype == "nothing") { return null; - } else if (objtype == "factor") { - const levels = handle.open("levels", { load: true }).values; - const ihandle = handle.open("values", { load: true }); - const codes = ihandle.values; - - const output = new Array(codes.length); - if ("missing-value-placeholder" in ihandle.attributes) { - const placeholder = ihandle.readAttribute("missing-value-placeholder").values[0]; - for (const [i, x] of codes.entries()) { - if (i == placeholder) { - output[i] = null; - } else { - output[i] = levels[x]; - } - } - } else { - for (const [i, x] of codes.entries()) { - output[i] = levels[x]; + } else if (objtype == "vector") { + const vectype = handle.readAttribute("uzuki_type").values[0]; + + if (vectype == "string") { + const vhandle = handle.open("data", { load: true }); + if (vhandle.attributes.indexOf("missing-value-placeholder") >= 0) { + const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0]; + return vhandle.values.map(x => { + if (x == placeholder) { + return null; + } else { + return x; + } + }); + } else { + return vhandle.values; } - } - return output; - - } else if (objtype = "string") { - const vhandle = handle.open("values", { load: true }); - if ("missing-value-placeholder" in vhandle.attributes) { - const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0]; - return vhandle.values.map(x => { - if (x == placeholder) { - return null; - } else { - return x; + } else if (vectype == "boolean") { + const vhandle = handle.open("data", { load: true }); + const values = vhandle.values; + + const output = new Array(values.length); + if (vhandle.attributes.indexOf("missing-value-placeholder") >= 0) { + const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0]; + for (const [i, x] of values.entries()) { + if (x == placeholder) { + output[i] = null; + } else { + output[i] = (x != 0); + } } - }); - } else { - return vhandle.values; - } - - } else if (objtype == "boolean") { - const vhandle = handle.open("values", { load: true }); - const values = vhandle.values; - - const output = new Array(values.length); - if ("missing-value-placeholder" in vhandle.attributes) { - const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0]; - for (const [i, x] of values.entries()) { - if (i == placeholder) { - output[i] = null; - } else { + } else { + for (const [i, x] of values.entries()) { output[i] = (x != 0); } } - } else { - for (const [i, x] of values.entries()) { - output[i] = (x != 0); - } - } - return output; + return output; + + } else if (vectype == "integer" || vectype == "number") { + const vhandle = handle.open("data", { load: true }); + let output = vhandle.values; - } else if (objtype = "integer" || objtype == "number") { - const vhandle = handle.open("values", { load: true }); - let output = vhandle.values; + if (vhandle.attributes.indexOf("missing-value-placeholder") >= 0) { + const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0]; + output = utils.substitutePlaceholder(output, placeholder); + } else if (vectype == "number") { + if (!(output instanceof Float64Array) && !(output instanceof Float32Array)) { + output = new Float64Array(output); + } + } - if ("missing-value-placeholder" in vhandle.attributes) { - const placeholder = vhandle.readAttribute("missing-value-placeholder").values[0]; - output = utils.substitutePlaceholder(output, placeholder); - } else if (objtype == "number") { - if (!(output instanceof Float64Array) && !(output instanceof Float32Array)) { - output = new Float64Array(output); + return output; + + } else if (vectype == "factor") { + const levels = handle.open("levels", { load: true }).values; + const ihandle = handle.open("data", { load: true }); + const codes = ihandle.values; + + const output = new Array(codes.length); + if (ihandle.attributes.indexOf("missing-value-placeholder") >= 0) { + const placeholder = ihandle.readAttribute("missing-value-placeholder").values[0]; + for (const [i, x] of codes.entries()) { + if (x == placeholder) { + output[i] = null; + } else { + output[i] = levels[x]; + } + } + } else { + for (const [i, x] of codes.entries()) { + output[i] = levels[x]; + } } - } - return output; + return output; + + } else { + console.warn("HDF5 simple list containing a vector of type '" + vectype + "' is not yet supported"); + return null; + } } else { - console.warn("JSON simple list containing type '" + objtype + "' is not yet supported"); + console.warn("HDF5 simple list containing type '" + objtype + "' is not yet supported"); return null; } } diff --git a/tests/readers/list.setup.R b/tests/readers/list.setup.R new file mode 100644 index 0000000..90eaf09 --- /dev/null +++ b/tests/readers/list.setup.R @@ -0,0 +1,55 @@ +library(alabaster.base) +PATH <- "objects" +dir.create(PATH, showWarnings=FALSE) +library(S4Vectors) + +{ + df <- list( + strings = "A", + integers = 1:3, + numbers = 4:6/2, + booleans = FALSE, + factors = factor("Z"), + nothing = NULL + ) + + path <- file.path(PATH, "list-basic") + unlink(path, recursive=TRUE) + dir.create(path) + + saveObject(df, file.path(path, "js")) + saveObject(df, file.path(path, "h5"), list.format='hdf5') +} + +{ + df <- list( + strings = c("a", NA_character_), + integers = c(1L, NA, 2L), + numbers = c(3.5, NA, 4.5), + booleans = c(NA, TRUE), + factors = factor(c("Z", NA, "A")) + ) + + path <- file.path(PATH, "list-missing") + unlink(path, recursive=TRUE) + dir.create(path) + + saveObject(df, file.path(path, "js")) + saveObject(df, file.path(path, "h5"), list.format='hdf5') +} + +{ + df <- list( + named = list(A = 1L, B = 2L), + unnamed = list("X", "Y", "Z"), + other = DataFrame(B = 2) + ) + + path <- file.path(PATH, "list-nested") + unlink(path, recursive=TRUE) + dir.create(path) + + saveObject(df, file.path(path, "js")) + saveObject(df, file.path(path, "h5"), list.format='hdf5') +} + diff --git a/tests/readers/list.test.js b/tests/readers/list.test.js new file mode 100644 index 0000000..ada9b86 --- /dev/null +++ b/tests/readers/list.test.js @@ -0,0 +1,51 @@ +import * as list from "../../src/readers/list.js"; +import { localNavigator } from "../utils.js"; +import * as path from "path"; +import * as scran from "scran.js"; + +beforeAll(async () => { await scran.initialize({ localFile: true }) }); +afterAll(async () => { await scran.terminate() }); + +const PATH = "objects"; + +test("basic list loading works as expected", async () => { + const basic_list = await list.readSimpleList(path.join(PATH, "list-basic", "js"), localNavigator); + expect(Object.keys(basic_list)).toEqual(["strings", "integers", "numbers", "booleans", "factors", "nothing" ]); + + expect(basic_list["strings"]).toEqual(["A"]); + expect(basic_list["integers"]).toEqual(new Int32Array([1,2,3])); + expect(basic_list["numbers"]).toEqual(new Float64Array([2,2.5,3])); + expect(basic_list["booleans"]).toEqual([false]); + expect(basic_list["factors"]).toEqual(["Z"]); + expect(basic_list["nothing"]).toBeNull(); + + const basic_list_h5 = await list.readSimpleList(path.join(PATH, "list-basic", "h5"), localNavigator); + expect(basic_list_h5).toEqual(basic_list); +}) + +test("list loading works with missing values", async () => { + const missing_list = await list.readSimpleList(path.join(PATH, "list-missing", "js"), localNavigator); + expect(Object.keys(missing_list)).toEqual(["strings", "integers", "numbers", "booleans", "factors" ]); + + expect(missing_list["strings"]).toEqual(["a", null]); + expect(missing_list["integers"]).toEqual([1,null,2]); + expect(missing_list["numbers"]).toEqual([3.5,null,4.5]); + expect(missing_list["booleans"]).toEqual([null, true]); + expect(missing_list["factors"]).toEqual(["Z", null, "A"]); + + const missing_list_h5 = await list.readSimpleList(path.join(PATH, "list-missing", "h5"), localNavigator); + expect(missing_list_h5).toEqual(missing_list); +}) + +test("list loading works with nesting", async () => { + const nested_list = await list.readSimpleList(path.join(PATH, "list-nested", "js"), localNavigator); + expect(Object.keys(nested_list)).toEqual(["named", "unnamed", "other"]); + + expect(nested_list["named"]).toEqual({ A: new Int32Array([1]), B: new Int32Array([2]) }); + expect(nested_list["unnamed"]).toEqual([["X"], ["Y"], ["Z"]]); + expect(nested_list["other"]).toBeNull(); + + const nested_list_h5 = await list.readSimpleList(path.join(PATH, "list-nested", "h5"), localNavigator); + expect(nested_list_h5).toEqual(nested_list); +}) +