Add ETL logic for MTB project data outside of the routine QUK17 project (#89)

This PR adds an ETL routine for MTB project data, that belong to a common research context.

This distinguishes from the ETL routine for routine MTB data registration, that is also submit to CentraXX.
Co-authored-by: wow-such-code <[email protected]>

Author: sven1103

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 # 2.0.0
 
+* Provides new ETL for MTB project data that are not supposed to be stored in QUK17 [(#89)](https://github.com/qbicsoftware/etl-scripts/pull/89)
 * Allow multiple sequencing lanes for MTB data
 
 # 1.9.0 2021-06-28

drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py

@@ -275,14 +275,19 @@ def get_last_exp_id(experiments):
     return exp_ids[-1]
 
 
-def getNextFreeBarcode(projectcode, numberOfBarcodes):
+def getNextFreeBarcode(projectcode, numberOfBarcodes, transaction, space):
     letters = string.ascii_uppercase
-    numberOfBarcodes += 1
-
-    currentLetter = letters[numberOfBarcodes / 999]
-    currentNumber = numberOfBarcodes % 999
-    code = projectcode + str(currentNumber).zfill(3) + currentLetter
-    return code + checksum.checksum(code)
+    sampleExists = True
+    newSampleCode = None
+    while sampleExists:
+        numberOfBarcodes += 1
+        currentLetter = letters[numberOfBarcodes / 999]
+        currentNumber = numberOfBarcodes % 999
+        code = projectcode + str(currentNumber).zfill(3) + currentLetter
+        newSampleCode = code + checksum.checksum(code)
+        sampleExists = transaction.getSampleForUpdate(
+            "/{space}/{sample}".format(space=space, sample=newSampleCode))
+    return newSampleCode
 
 
 def register_rnaseq(rna_seq_files, transaction):
@@ -320,7 +325,10 @@ def register_rnaseq(rna_seq_files, transaction):
     sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(pc))
     result = search_service.searchForSamples(sc)
     print("Found {} samples for project {} in space {}.".format(len(result), project, space))
-    new_rna_sample_barcode = getNextFreeBarcode(project, numberOfBarcodes=len(result))
+    new_rna_sample_barcode = getNextFreeBarcode(project,
+                                                numberOfBarcodes=len(result),
+                                                transaction=transaction,
+                                                space=space)
 
     # Now get the parent sample id (tumor sample, type: BIOLOGICAL_SAMPLE)
     tumor_dna_sample = getsample(dna_barcode, transaction)

drop-boxes/register-mtb-projects-dropbox/mtbutils.py

@@ -0,0 +1,30 @@
+""" A utility class, holding helper functions
+for the main dropbox """
+
+import datetime
+import subprocess as sp
+
+MTB_CONVERTER_PATH = '/home/qeana10/bin/miniconda/bin/mtbconverter'
+
+def mtbconverter(cmds):
+    """Tries to activate a given conda environment"""
+    command = [MTB_CONVERTER_PATH] + cmds
+    ret_code = sp.call(command)
+    return ret_code
+
+def log_stardate(msg):
+    """Prints a message nicely with current stardate"""
+    stardate = datetime.datetime.now()
+    return '{} [{}]: {}'.format(stardate.isoformat(), 'mtbconverter', msg)
+
+class MTBdropboxerror(Exception):
+    """A generic Exception class for this dropbox."""
+
+class Counter():
+
+    def __init__(self):
+        self.counter = 1
+    
+    def newId(self):
+        self.counter += 1
+        return self.counter - 1

drop-boxes/register-mtb-projects-dropbox/plugin.properties

@@ -0,0 +1,11 @@
+#
+# Drop box for registering a fastq file as a data set
+#
+# Variables:
+#   incoming-root-dir
+#     Path to the directory which contains incoming directories for drop boxes.
+incoming-dir = ${incoming-root-dir}/QBiC-register-mtb-projects-data
+incoming-data-completeness-condition = marker-file
+top-level-data-set-handler = ch.systemsx.cisd.etlserver.registrator.api.v2.JythonTopLevelDataSetHandlerV2
+script-path = register-mtb-projects-dropbox.py
+storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor