Release 1.10.0

Release 1.10.0

Author: sven1103

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+# 1.10.0 2021-07-26
+
+* Provides new ETL for MTB project data that are not supposed to be stored in QUK17 [(#89)](https://github.com/qbicsoftware/etl-scripts/pull/89)
+* Allow multiple sequencing lanes for MTB data
+
 # 1.9.0 2021-06-28
 
 * Provides new ETL routine written in Java, that will replace all Jython scripts at some point [(#85)](https://github.com/qbicsoftware/etl-scripts/pull/85)

drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py

@@ -108,6 +108,8 @@
 
 # Regex matching the RNAseq sample file naming specification
 RNASEQ_REG = re.compile(r'.*tumor_rna.[1,2]{1}.fastq.gz')
+# Update from 2021-07-06: Support lanes, indicated with a three digit number
+RNASEQ_REG_LANES = re.compile(r'.*tumor_rna_[0-9]{3}.[1,2]{1}.fastq.gz')
 
 # Path to the openBIS properties file
 PROPERTIES = '/etc/openbis.properties'
@@ -214,7 +216,7 @@ def process(transaction):
     for in_file in file_list:
         if in_file.endswith('origlabfilename') or in_file.endswith('sha256sum') or 'source_dropbox.txt' in in_file:
             continue
-        if RNASEQ_REG.findall(in_file):
+        if RNASEQ_REG.findall(in_file) or RNASEQ_REG_LANES.findall(in_file):
             rna_seq_files.append(in_file)
         elif 'fastq' in in_file:
             if 'normal' in in_file:
@@ -253,8 +255,10 @@ def execute_vcf_registration(vcf_files, transaction):
 
 
 def execute_fastq_registration(fastqs_normal, fastqs_tumor, transaction):
-   if len(fastqs_tumor) != 2 or len(fastqs_normal) != 2:
+   if len(fastqs_tumor) < 2 or len(fastqs_normal) < 2:
         raise mtbutils.MTBdropboxerror("Tumor/normal fastq dataset was not complete. Please check.")
+   elif len(fastqs_tumor) != len(fastqs_normal):
+        raise mtbutils.MTBdropboxerror("Tumor/normal fastq dataset dont have the same number of files. Are all lanes provided?")
    else:
         proc_fastq(fastqs_tumor, transaction)
         proc_fastq(fastqs_normal, transaction)
@@ -271,14 +275,19 @@ def get_last_exp_id(experiments):
     return exp_ids[-1]
 
 
-def getNextFreeBarcode(projectcode, numberOfBarcodes):
+def getNextFreeBarcode(projectcode, numberOfBarcodes, transaction, space):
     letters = string.ascii_uppercase
-    numberOfBarcodes += 1
-
-    currentLetter = letters[numberOfBarcodes / 999]
-    currentNumber = numberOfBarcodes % 999
-    code = projectcode + str(currentNumber).zfill(3) + currentLetter
-    return code + checksum.checksum(code)
+    sampleExists = True
+    newSampleCode = None
+    while sampleExists:
+        numberOfBarcodes += 1
+        currentLetter = letters[numberOfBarcodes / 999]
+        currentNumber = numberOfBarcodes % 999
+        code = projectcode + str(currentNumber).zfill(3) + currentLetter
+        newSampleCode = code + checksum.checksum(code)
+        sampleExists = transaction.getSampleForUpdate(
+            "/{space}/{sample}".format(space=space, sample=newSampleCode))
+    return newSampleCode
 
 
 def register_rnaseq(rna_seq_files, transaction):
@@ -296,7 +305,8 @@ def register_rnaseq(rna_seq_files, transaction):
                         the reason for the failure.
     """
     print(mtbutils.log_stardate('Registering incoming MTB RNAseq data {}'.format(rna_seq_files)))
-    assert len(rna_seq_files) == 2
+    # Check if dataset files are paired end and complete
+    assert len(rna_seq_files) % 2 == 0
     file1 = os.path.basename(rna_seq_files[0])
     file2 = os.path.basename(rna_seq_files[1])
     assert len(set(QCODE_REG.findall(file1))) == 1
@@ -315,7 +325,10 @@ def register_rnaseq(rna_seq_files, transaction):
     sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(pc))
     result = search_service.searchForSamples(sc)
     print("Found {} samples for project {} in space {}.".format(len(result), project, space))
-    new_rna_sample_barcode = getNextFreeBarcode(project, numberOfBarcodes=len(result))
+    new_rna_sample_barcode = getNextFreeBarcode(project,
+                                                numberOfBarcodes=len(result),
+                                                transaction=transaction,
+                                                space=space)
 
     # Now get the parent sample id (tumor sample, type: BIOLOGICAL_SAMPLE)
     tumor_dna_sample = getsample(dna_barcode, transaction)
@@ -448,7 +461,7 @@ def proc_fastq(fastq_file, transaction):
     """Register fastq as dataset in openBIS"""
 
     # Check, if there are file pairs present (paired-end data!)
-    if len(fastq_file) != 2:
+    if len(fastq_file) % 2 != 0:
         raise mtbutils.MTBdropboxerror('Expecting paired end reads files, found only {}'
             .format(len(fastq_file)))
     qbiccode_f1 = QCODE_REG.findall(os.path.basename(fastq_file[0]))

drop-boxes/register-mtb-projects-dropbox/mtbutils.py

@@ -0,0 +1,30 @@
+""" A utility class, holding helper functions
+for the main dropbox """
+
+import datetime
+import subprocess as sp
+
+MTB_CONVERTER_PATH = '/home/qeana10/bin/miniconda/bin/mtbconverter'
+
+def mtbconverter(cmds):
+    """Tries to activate a given conda environment"""
+    command = [MTB_CONVERTER_PATH] + cmds
+    ret_code = sp.call(command)
+    return ret_code
+
+def log_stardate(msg):
+    """Prints a message nicely with current stardate"""
+    stardate = datetime.datetime.now()
+    return '{} [{}]: {}'.format(stardate.isoformat(), 'mtbconverter', msg)
+
+class MTBdropboxerror(Exception):
+    """A generic Exception class for this dropbox."""
+
+class Counter():
+
+    def __init__(self):
+        self.counter = 1
+    
+    def newId(self):
+        self.counter += 1
+        return self.counter - 1

drop-boxes/register-mtb-projects-dropbox/plugin.properties

@@ -0,0 +1,11 @@
+#
+# Drop box for registering a fastq file as a data set
+#
+# Variables:
+#   incoming-root-dir
+#     Path to the directory which contains incoming directories for drop boxes.
+incoming-dir = ${incoming-root-dir}/QBiC-register-mtb-projects-data
+incoming-data-completeness-condition = marker-file
+top-level-data-set-handler = ch.systemsx.cisd.etlserver.registrator.api.v2.JythonTopLevelDataSetHandlerV2
+script-path = register-mtb-projects-dropbox.py
+storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor