Bug fixes

hubmapconsortium · Oct 31, 2024 · e342c4a · e342c4a
1 parent 75a9f01
commit e342c4a
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 5 deletions.
diff --git a/src/combine-metadata.js b/src/combine-metadata.js
@@ -62,14 +62,25 @@ async function readMetadata(item, config) {
   }
 }
 
+function stringifyFields(obj, fields) {
+  const shouldStringify = (value) => typeof value === 'object' && value !== null;
+  const result = {};
+  for (const field of fields) {
+    const value = obj[field];
+    result[field] = shouldStringify(value) ? JSON.stringify(value) : value;
+  }
+
+  return result;
+}
+
 async function main() {
   const config = getConfig();
   const maxConcurrency = config.get(MAX_CONCURRENCY, DEFAULT_MAX_CONCURRENCY);
   const summaries = await DatasetSummaries.load(getSummariesFilePath(config));
   const metadata = await concurrentMap(Array.from(summaries.values()), (item) => readMetadata(item, config), {
     maxConcurrency,
   });
-  const items = metadata.filter((item) => !!item);
+  const items = metadata.filter((item) => !!item).map((item) => stringifyFields(item, METADATA_FIELDS));
 
   const outputFile = join(getOutputDir(config), 'bulk-dataset-metadata.csv');
   const content = Papa.unparse({

diff --git a/src/slurm/slurm-annotate-worker.sh b/src/slurm/slurm-annotate-worker.sh
@@ -6,10 +6,10 @@
 #SBATCH -o slurm-output/annotate-worker/hra-run_%j.txt
 #SBATCH -e slurm-output/annotate-worker/hra-run_%j.err
 #SBATCH --mail-type=FAIL,TIME_LIMIT,INVALID_DEPEND,ARRAY_TASKS
-#SBATCH --ntasks=4
+#SBATCH --ntasks=1
 #SBATCH --ntasks-per-core=1
-#SBATCH --time=1:20:00
-#SBATCH --mem=128G
+#SBATCH --time=2:00:00
+#SBATCH --mem=64G
 #SBATCH --kill-on-invalid-dep=yes
 
 module load python/3.10.5
@@ -18,7 +18,7 @@ module load singularity
 # ---------------------------------------
 # Constants
 # ---------------------------------------
-readonly NUM_TASKS="${SLURM_NTASKS:-4}"
+readonly NUM_TASKS="${SLURM_NTASKS:-1}"
 readonly HORIZONTAL_LINE='------------------------------------------------'
 
 readonly CWL_PIPELINE_URL="https://cdn.jsdelivr.net/gh/hubmapconsortium/hra-workflows@main/pipeline.cwl"
@@ -126,6 +126,10 @@ run_job() {
   # Create temp dir
   tmp_dir="$(mktemp -d -p "$CWL_TMP_DIR" "$index-XXXX")"
 
+  # Make a temp dir for apptainer
+  export APPTAINER_TMPDIR="${tmp_dir}-apptainer/"
+  mkdir -p "${APPTAINER_TMPDIR}"
+
   # Build cwl-runner command arguments
   args+=(--singularity --no-doc-cache)
   args+=(--tmpdir-prefix "$tmp_dir/") # Note: prefix must end with a '/'