fix #3389 (#3450)

* fix #3389

* add coverages

* update html [no ci]

* no copyfile

* missing [ - no ci

* Dowload -> Download

Author: antgonza

qiita_pet/handlers/download.py

@@ -9,7 +9,7 @@
 from tornado.web import authenticated, HTTPError
 from tornado.gen import coroutine
 
-from os.path import basename, getsize, join, isdir
+from os.path import basename, getsize, join, isdir, getctime
 from os import walk
 
 from .base_handlers import BaseHandler
@@ -23,7 +23,7 @@
 from qiita_db.util import (filepath_id_to_rel_path, get_db_files_base_dir,
                            get_filepath_information, get_mountpoint,
                            filepath_id_to_object_id, get_data_types,
-                           retrieve_filepaths)
+                           retrieve_filepaths, get_work_base_dir)
 from qiita_db.meta_util import validate_filepath_access_by_user
 from qiita_db.metadata_template.sample_template import SampleTemplate
 from qiita_db.metadata_template.prep_template import PrepTemplate
@@ -35,6 +35,9 @@
 from uuid import uuid4
 from base64 import b64encode
 from datetime import datetime, timedelta, timezone
+from tempfile import mkdtemp
+from zipfile import ZipFile
+from io import BytesIO
 
 
 class BaseHandlerDownload(BaseHandler):
@@ -374,6 +377,138 @@ def get(self, path):
         self.finish()
 
 
+class DownloadDataReleaseFromPrep(BaseHandlerDownload):
+    @authenticated
+    @coroutine
+    @execute_as_transaction
+    def get(self, prep_template_id):
+        """ This method constructs an on the fly ZIP with all the files
+            required for a data-prep release/data-delivery. Mainly sample, prep
+            info, bioms and coverage
+        """
+        user = self.current_user
+        if user.level not in ('admin', 'web-lab admin'):
+            raise HTTPError(403, reason="%s doesn't have access to download "
+                            "the data release files" % user.email)
+
+        pid = int(prep_template_id)
+        pt = PrepTemplate(pid)
+        sid = pt.study_id
+        st = SampleTemplate(sid)
+        date = datetime.now().strftime('%m%d%y-%H%M%S')
+        td = mkdtemp(dir=get_work_base_dir())
+
+        files = []
+        readme = [
+            f'Delivery created on {date}',
+            '',
+            f'Host (human) removal: {pt.artifact.human_reads_filter_method}',
+            '',
+            # this is not changing in the near future so just leaving
+            # hardcoded for now
+            'Main woltka reference: WoLr2, more info visit: '
+            'https://ftp.microbio.me/pub/wol2/',
+            '',
+            f"Qiita's prep: https://qiita.ucsd.edu/study/description/{sid}"
+            f"?prep_id={pid}",
+            '',
+        ]
+
+        # helper dict to add "user/human" friendly names to the bioms
+        human_names = {
+            'ec.biom': 'KEGG Enzyme (EC)',
+            'per-gene.biom': 'Per gene Predictions',
+            'none.biom': 'Per genome Predictions',
+            'cell_counts.biom': 'Cell counts',
+            'pathway.biom': 'KEGG Pathway',
+            'ko.biom': 'KEGG Ontology (KO)',
+            'rna_copy_counts.biom': 'RNA copy counts'
+        }
+
+        # sample-info creation
+        fn = join(td, f'sample_information_from_prep_{pid}.tsv')
+        readme.append(f'Sample information: {basename(fn)}')
+        files.append([fn, basename(fn)])
+        st.to_dataframe(samples=list(pt)).to_csv(fn, sep='\t')
+
+        # prep-info creation
+        fn = join(td, f'prep_information_{pid}.tsv')
+        readme.append(f'Prep information: {basename(fn)}')
+        files.append([fn, basename(fn)])
+        pt.to_dataframe().to_csv(fn, sep='\t')
+
+        readme.append('')
+
+        # finding the bioms to be added
+        bioms = dict()
+        coverages = None
+        for a in Study(sid).artifacts(artifact_type='BIOM'):
+            if a.prep_templates[0].id != pid:
+                continue
+            biom = None
+            for fp in a.filepaths:
+                if fp['fp_type'] == 'biom':
+                    biom = fp
+                if coverages is None and 'coverages.tgz' == basename(fp['fp']):
+                    coverages = fp['fp']
+            if biom is None:
+                continue
+            biom_fn = basename(biom['fp'])
+            # there is a small but real chance that the same prep has the same
+            # artifacts so using the latests
+            if biom_fn not in bioms:
+                bioms[biom_fn] = [a, biom]
+            else:
+                if getctime(biom['fp']) > getctime(bioms[biom_fn][1]['fp']):
+                    bioms[biom_fn] = [a, biom]
+
+        # once we have all the bioms, we can add them to the list of zips
+        # and to the readme the biom details and all the processing
+        for fn, (a, fp) in bioms.items():
+            aname = basename(fp["fp"])
+            nname = f'{a.id}_{aname}'
+            files.append([fp['fp'], nname])
+
+            hname = ''
+            if aname in human_names:
+                hname = human_names[aname]
+            readme.append(f'{nname}\t{hname}')
+
+            for an in set(a.ancestors.nodes()):
+                p = an.processing_parameters
+                if p is not None:
+                    c = p.command
+                    cn = c.name
+                    s = c.software
+                    sn = s.name
+                    sv = s.version
+                    pd = p.dump()
+                    readme.append(f'\t{cn}\t{sn}\t{sv}\t{pd}')
+
+        # if a coverage was found, add it to the list of files
+        if coverages is not None:
+            fn = basename(coverages)
+            readme.append(f'{fn}\tcoverage files')
+            files.append([coverages, fn])
+
+        fn = join(td, 'README.txt')
+        with open(fn, 'w') as fp:
+            fp.write('\n'.join(readme))
+        files.append([fn, basename(fn)])
+
+        zp_fn = f'data_release_{pid}_{date}.zip'
+        zp = BytesIO()
+        with ZipFile(zp, 'w') as zipf:
+            for fp, fn in files:
+                zipf.write(fp, fn)
+
+        self.set_header('Content-Type', 'application/zip')
+        self.set_header("Content-Disposition", f"attachment; filename={zp_fn}")
+        self.write(zp.getvalue())
+        zp.close()
+        self.finish()
+
+
 class DownloadPublicHandler(BaseHandlerDownload):
     @coroutine
     @execute_as_transaction

qiita_pet/support_files/doc/source/faq.rst

@@ -316,7 +316,7 @@ To take advantage of this feature you need to:
    server that is OK to give access to the key created to your storage-shed; note that if
    you want to completely stop that key to work you can open that file and remove the line
    with the name of this key.
-#. Dowload your new generated key `qiita-key` (the file) to your local computer and use it
+#. Download your new generated key `qiita-key` (the file) to your local computer and use it
    in the `Key` option of "Upload via Remote Server (ADVANCED)".
 
 Using this key you can `List Files` to test the connection and verify the list of study files. Then,

qiita_pet/templates/study_ajax/prep_summary.html

@@ -447,6 +447,11 @@ <h4>
       {% end %}
       {% if editable %}
         <br/>
+        {% if user_level in ('admin', 'wet-lab admin') and data_type in {'Metagenomic', 'Metatranscriptomic'} %}
+            <button class="btn btn-info" onclick="this.disabled=true; window.location='{% raw qiita_config.portal_dir %}/download_data_release_from_prep/{{prep_id}}';">
+                <span class="glyphicon glyphicon-download-alt"></span> Download Data Release
+            </button>
+        {% end %}
         {% if deprecated %}
           <a class="btn btn-warning" onclick="deprecate_preparation({{prep_id}}, false);"><span class="glyphicon glyphicon-pushpin"></span> Remove Deprecation</a>
         {% else%}

qiita_pet/webserver.py

@@ -55,7 +55,8 @@
     DownloadHandler, DownloadStudyBIOMSHandler, DownloadRelease,
     DownloadRawData, DownloadEBISampleAccessions, DownloadEBIPrepAccessions,
     DownloadUpload, DownloadPublicHandler, DownloadPublicArtifactHandler,
-    DownloadSampleInfoPerPrep, DownloadPrivateArtifactHandler)
+    DownloadSampleInfoPerPrep, DownloadPrivateArtifactHandler,
+    DownloadDataReleaseFromPrep)
 from qiita_pet.handlers.prep_template import (
     PrepTemplateHandler, PrepTemplateGraphHandler, PrepTemplateJobHandler)
 from qiita_pet.handlers.ontology import OntologyHandler
@@ -194,6 +195,8 @@ def __init__(self):
             (r"/software/", SoftwareHandler),
             (r"/workflows/", WorkflowsHandler),
             (r"/download/(.*)", DownloadHandler),
+            (r"/download_data_release_from_prep/(.*)",
+                DownloadDataReleaseFromPrep),
             (r"/download_study_bioms/(.*)", DownloadStudyBIOMSHandler),
             (r"/download_raw_data/(.*)", DownloadRawData),
             (r"/download_ebi_accessions/samples/(.*)",