Dev updates 2021 (#3063)

* rm key

* use test_ssh_key

* flake8

* add chmod

* ssh_key -> ssh_key_fp

* fix #3057

* adding JS for download

* minor improvements

* addressing @ElDeveloper comment

* fix #3049

* fix #3060

* flake8

* addressing @ElDeveloper comments

* Delete 81.sql

* No

* Yes

* No

* Yes

* Replace qiita-spots to qiita

Co-authored-by: Antonio Gonzalez <[email protected]>

Author: ElDeveloper

README.rst

@@ -19,12 +19,12 @@ typically limiting for researchers studying microbial ecology.
 
 Qiita is currently in beta status. We are very open to community
 contributions and feedback. If you're interested in contributing to Qiita,
-see `CONTRIBUTING.md <https://github.com/biocore/qiita/blob/master/CONTRIBUTING.md>`__.
+see `CONTRIBUTING.md <https://github.com/qiita-spots/qiita/blob/master/CONTRIBUTING.md>`__.
 If you'd like to report bugs or request features, you can do that in the
-`Qiita issue tracker <https://github.com/biocore/qiita/issues>`__.
+`Qiita issue tracker <https://github.com/qiita-spots/qiita/issues>`__.
 
 To install and configure your own Qiita server, see
-`INSTALL.md <https://github.com/biocore/qiita/blob/master/INSTALL.md>`__.
+`INSTALL.md <https://github.com/qiita-spots/qiita/blob/master/INSTALL.md>`__.
 
 For more specific details about qiita visit `the Qiita main site tutorial <https://qiita.microbio.me/static/doc/html/qiita-philosophy/index.html>`__.
 
@@ -71,9 +71,7 @@ future.
   existing studies by submitting a fix proposals to the authors of the study.
 
 
-.. |Build Status| image:: https://travis-ci.org/biocore/qiita.png?branch=master
-   :target: https://travis-ci.org/biocore/qiita
-.. |Coverage Status| image:: https://codecov.io/gh/biocore/qiita/branch/master/graph/badge.svg
-   :target: https://codecov.io/gh/biocore/qiita
-.. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg
-   :target: https://gitter.im/biocore/qiita?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+.. |Build Status| image:: https://travis-ci.org/qiita-spots/qiita.png?branch=master
+   :target: https://travis-ci.org/qiita-spots/qiita
+.. |Coverage Status| image:: https://codecov.io/gh/qiita-spots/qiita/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/qiita-spots/qiita

qiita_db/metadata_template/base_metadata_template.py

@@ -558,20 +558,20 @@ def _clean_validate_template(cls, md_template, study_id,
         error = []
         if pgsql_reserved:
             error.append(
-                "The following column names in the template contain PgSQL "
-                "reserved words: %s." % ", ".join(pgsql_reserved))
+                "These column names are PgSQL reserved words, replace them: "
+                "~~ %s ~~." % ", ".join(pgsql_reserved))
         if invalid:
             error.append(
-                "The following column names in the template contain invalid "
-                "chars: %s." % ", ".join(invalid))
+                "These column names contain invalid chars, remove or replace "
+                "them: ~~ %s ~~." % ", ".join(invalid))
         if forbidden:
             error.append(
-                "The following column names in the template contain invalid "
-                "values: %s." % ", ".join(forbidden))
+                "These column names are not valid in this information file, "
+                "remove them: ~~ %s ~~." % ", ".join(forbidden))
         if qiime2_reserved:
             error.append(
-                "The following column names in the template contain QIIME2 "
-                "reserved words: %s." % ", ".join(pgsql_reserved))
+                "These columns are QIIME2 reserved words, replace them: "
+                " ~~ %s ~~." % ", ".join(pgsql_reserved))
 
         if error:
             raise qdb.exceptions.QiitaDBColumnError(
@@ -1160,13 +1160,15 @@ def to_file(self, fp, samples=None):
             df.to_csv(fp, index_label='sample_name', na_rep="", sep='\t',
                       encoding='utf-8')
 
-    def _common_to_dataframe_steps(self):
+    def _common_to_dataframe_steps(self, samples=None):
         """Perform the common to_dataframe steps
 
         Returns
         -------
         pandas DataFrame
             The metadata in the template,indexed on sample id
+        samples list of string, optional
+            A list of the sample names we actually want to retrieve
         """
         with qdb.sql_connection.TRN:
             # Retrieve all the information from the database
@@ -1175,7 +1177,12 @@ def _common_to_dataframe_steps(self):
                      FROM qiita.{0}
                      WHERE sample_id != '{1}'""".format(
                         self._table_name(self._id), QIITA_COLUMN_NAME)
-            qdb.sql_connection.TRN.add(sql)
+            if samples is None:
+                qdb.sql_connection.TRN.add(sql)
+            else:
+                sql += ' AND sample_id IN %s'
+                qdb.sql_connection.TRN.add(sql, [tuple(samples)])
+
             # this query is going to return a tuple
             # (sample_id, dict of columns/values); however it's important to
             # notice that we can't assure that all column/values pairs are the

qiita_db/metadata_template/constants.py

@@ -17,6 +17,7 @@
     'EBI': Restriction(columns={'collection_timestamp': datetime,
                                 'physical_specimen_location': str,
                                 'taxon_id': int,
+                                'description': str,
                                 'scientific_name': str},
                        error_msg="EBI submission disabled"),
     # The following columns are required for the official main QIITA site

qiita_db/metadata_template/sample_template.py

@@ -333,15 +333,17 @@ def biosample_accessions(self, value):
         """
         self._update_accession_numbers('biosample_accession', value)
 
-    def to_dataframe(self, add_ebi_accessions=False):
+    def to_dataframe(self, add_ebi_accessions=False, samples=None):
         """Returns the metadata template as a dataframe
 
         Parameters
         ----------
         add_ebi_accessions : bool, optional
             If this should add the ebi accessions
+        samples list of string, optional
+            A list of the sample names we actually want to retrieve
         """
-        df = self._common_to_dataframe_steps()
+        df = self._common_to_dataframe_steps(samples=samples)
 
         if add_ebi_accessions:
             accessions = self.ebi_sample_accessions

qiita_db/metadata_template/test/test_sample_template.py

@@ -1963,8 +1963,7 @@ def test_to_dataframe(self):
                '1.SKM4.640180', '1.SKM5.640177', '1.SKM6.640187',
                '1.SKM7.640188', '1.SKM8.640201', '1.SKM9.640192'}
         self.assertEqual(set(obs.index), exp)
-
-        self.assertEqual(set(obs.columns), {
+        exp_columns = {
             'physical_specimen_location', 'physical_specimen_remaining',
             'dna_extracted', 'sample_type', 'collection_timestamp',
             'host_subject_id', 'description', 'latitude', 'longitude',
@@ -1973,7 +1972,15 @@ def test_to_dataframe(self):
             'water_content_soil', 'elevation', 'temp', 'tot_nitro',
             'samp_salinity', 'altitude', 'env_biome', 'country', 'ph',
             'anonymized_name', 'tot_org_carb', 'description_duplicate',
-            'env_feature', 'scientific_name', 'qiita_study_id'})
+            'env_feature', 'scientific_name', 'qiita_study_id'}
+        self.assertEqual(set(obs.columns), exp_columns)
+
+        # test limiting samples produced
+        exp_samples = set(['1.SKD4.640185', '1.SKD5.640186'])
+        obs = self.tester.to_dataframe(samples=exp_samples)
+        self.assertEqual(len(obs), 2)
+        self.assertEqual(set(obs.index), exp_samples)
+        self.assertEqual(set(obs.columns), exp_columns)
 
         # test with add_ebi_accessions as True
         obs = self.tester.to_dataframe(True)

qiita_db/processing_job.py

@@ -1763,8 +1763,8 @@ def shape(self):
         if self.command.name == 'Validate':
             # Validate only has two options to calculate it's size: template (a
             # job that has a preparation linked) or analysis (is from an
-            # analysis).
-            if 'template' in parameters:
+            # analysis). However, 'template' can be present and be None
+            if 'template' in parameters and parameters['template'] is not None:
                 try:
                     pt = qdb.metadata_template.prep_template.PrepTemplate(
                         parameters['template'])

qiita_pet/handlers/artifact_handlers/base_handlers.py

@@ -160,7 +160,7 @@ def artifact_summary_get_request(user, artifact_id):
             buttons.append(btn_base % ('revert to sandbox', 'sandbox',
                                        'Revert to sandbox'))
 
-        if user.level == 'admin':
+        if user.level == 'admin' and not study.autoloaded:
             if artifact.can_be_submitted_to_ebi:
                 buttons.append(
                     '<a class="btn btn-primary btn-sm" '

qiita_pet/handlers/artifact_handlers/tests/test_base_handlers.py

@@ -257,6 +257,22 @@ def test_artifact_summary_get_request(self):
                'errored_summary_jobs': []}
         self.assertEqual(obs, exp)
 
+        # the buttons shouldn't be present when the study is autoloaded
+        study = a.study
+        study.autoloaded = True
+        exp['buttons'] = ('<button onclick="if (confirm(\'Are you sure you '
+                          'want to make public artifact id: 2?\')) { '
+                          'set_artifact_visibility(\'public\', 2) }" '
+                          'class="btn btn-primary btn-sm">Make public'
+                          '</button> <button onclick="if (confirm(\'Are you '
+                          'sure you want to revert to sandbox artifact id: '
+                          '2?\')) { set_artifact_visibility(\'sandbox\', 2) '
+                          '}" class="btn btn-primary btn-sm">Revert to '
+                          'sandbox</button> ' + private_download_button % 2)
+        obs = artifact_summary_get_request(User('[email protected]'), 2)
+        self.assertEqual(obs, exp)
+        study.autoloaded = False
+
         # analysis artifact
         obs = artifact_summary_get_request(user, 8)
         exp = {'name': 'noname',

qiita_pet/handlers/download.py

@@ -46,10 +46,7 @@ def _check_permissions(self, sid):
                 study_info['message'], self.current_user.email, sid))
         return Study(sid)
 
-    def _generate_files(self, header_name, accessions, filename):
-        text = "sample_name\t%s\n%s" % (header_name, '\n'.join(
-            ["%s\t%s" % (k, v) for k, v in accessions.items()]))
-
+    def _finish_generate_files(self, filename, text):
         self.set_header('Content-Description', 'text/csv')
         self.set_header('Expires', '0')
         self.set_header('Cache-Control', 'no-cache')
@@ -58,6 +55,12 @@ def _generate_files(self, header_name, accessions, filename):
         self.write(text)
         self.finish()
 
+    def _generate_files(self, header_name, accessions, filename):
+        text = "sample_name\t%s\n%s" % (header_name, '\n'.join(
+            ["%s\t%s" % (k, v) for k, v in accessions.items()]))
+
+        self._finish_generate_files(filename, text)
+
     def _list_dir_files_nginx(self, dirpath):
         """Generates a nginx list of files in the given dirpath for nginx
 
@@ -327,6 +330,25 @@ def get(self, prep_template_id):
             'ebi_experiment_accessions_study_%s_prep_%s.tsv' % (sid, pid))
 
 
+class DownloadSampleInfoPerPrep(BaseHandlerDownload):
+    @authenticated
+    @coroutine
+    @execute_as_transaction
+    def get(self, prep_template_id):
+        pid = int(prep_template_id)
+        pt = PrepTemplate(pid)
+        sid = pt.study_id
+
+        self._check_permissions(sid)
+
+        st = SampleTemplate(sid)
+
+        text = st.to_dataframe(samples=list(pt)).to_csv(None, sep='\t')
+
+        self._finish_generate_files(
+            'sample_information_from_prep_%s.tsv' % pid, text)
+
+
 class DownloadUpload(BaseHandlerDownload):
     @authenticated
     @coroutine

qiita_pet/handlers/upload.py

@@ -9,7 +9,7 @@
 from tornado.web import authenticated, HTTPError
 
 from os.path import join, exists
-from os import remove
+from os import remove, chmod
 from json import loads, dumps
 
 from collections import defaultdict
@@ -142,6 +142,7 @@ def post(self, study_id):
 
         with open(ssh_key_fp, 'wb') as f:
             f.write(ssh_key)
+        chmod(ssh_key_fp, 0o600)
 
         qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
         if method == 'list':

qiita_pet/templates/study_ajax/prep_summary.html

@@ -429,6 +429,7 @@ <h4>
       <span id="prep-name-span">{{name}}</span> - ID {{prep_id}} ({{data_type}})
       <a class="btn btn-default" data-toggle="modal" data-target="#update-prep-name"><span class="glyphicon glyphicon-pencil"></span> Edit name</a>
       <a class="btn btn-default" href="{% raw qiita_config.portal_dir %}/download/{{download_prep_id}}"><span class="glyphicon glyphicon-download-alt"></span> Prep info</a>
+      <a class="btn btn-default" href="{% raw qiita_config.portal_dir %}/download_sample_info_per_prep/{{prep_id}}"><span class="glyphicon glyphicon-download-alt"></span> Sample info (only this prep)</a>
       {% if is_submitted_to_ebi %}
         <a class="btn btn-default" href="{% raw qiita_config.portal_dir %}/download_ebi_accessions/experiments/{{prep_id}}"><span class="glyphicon glyphicon-download-alt"></span> EBI experiment accessions</a>
       {% end %}

qiita_pet/templates/study_ajax/sample_prep_summary.html

@@ -52,6 +52,44 @@
       sampleTemplatePage.$refs.stElem.deleteSamples(to_delete);
     }
   }
+
+  /**
+   *
+   * Function to download the data displayed in the page
+   *
+   */
+  function download_summary(){
+    var information = [];
+    var prep_names = {% raw columns %};
+
+    // adding header
+    var header = ['sample_id']
+    $.each(prep_colums, function(i, prep){
+      header.push(prep_names[prep]);
+    });
+    information.push(header.join('\t'))
+
+    // adding data
+    $.each(rows, function (i, row){
+      var line = [row['sample']]
+      $.each(prep_colums, function(i, prep){
+        line.push(row[prep]);
+      });
+      information.push(line.join('\t'));
+    });
+
+    information = information.join('\n');
+
+    // downloading file
+    var element = document.createElement('a');
+    element.setAttribute('href', 'data:text/plain;charset=utf-8,' + encodeURIComponent(information));
+    element.setAttribute('download', 'study_{{study_id}}_sample_preparations_summary.tsv');
+    element.style.display = 'none';
+    document.body.appendChild(element);
+    element.click();
+    document.body.removeChild(element);
+  }
+
   var columns = [
       { 'name': '<input type="checkbox" onchange="toggleCheckboxes(this)">',
         'field': 'sample-delete', 'id': 'sample-delete', width: 69, sortable: false, formatter: linkFormatter },
@@ -271,6 +309,9 @@
     <tr>
       <td>
         <h3>Sample Summary</h3><br/>
+        <button class="btn btn-default st-interactive" onclick="download_summary();">
+            <span class="glyphicon glyphicon-download-alt"></span> Summary
+        </button>
         <button class="btn btn-danger st-interactive" onclick="deleteIndividualSamples()">
             <span class="glyphicon glyphicon-trash"></span> Delete Selected
         </button>
@@ -280,6 +321,7 @@ <h3>Sample Summary</h3><br/>
       </td>
       <td style="text-align: right">
         Add sample column information to table
+        <br/>
         <select id="metadata_category" name="metadata_category">
           <option value=""></option>
           {% for col in categories %}

qiita_pet/test/test_download.py

@@ -6,12 +6,14 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
+import pandas as pd
 from unittest import main
 from mock import Mock
 from os.path import exists, isdir, join, basename
 from os import remove, makedirs, close
 from shutil import rmtree
 from tempfile import mkdtemp, mkstemp
+from io import StringIO
 
 from biom.util import biom_open
 from biom import example_table as et
@@ -340,6 +342,25 @@ def test_download(self):
         self.assertEqual(response.code, 405)
 
 
+class TestDownloadSampleInfoPerPrep(TestHandlerBase):
+
+    def test_download(self):
+        # check success
+        response = self.get('/download_sample_info_per_prep/1')
+        self.assertEqual(response.code, 200)
+
+        df = pd.read_csv(StringIO(response.body.decode('ascii')), sep='\t')
+        # just testing shape as the actual content is tested in the dataframe
+        # generation
+        self.assertEqual(df.shape, (27, 33))
+
+        # changing user so we can test the failures
+        BaseHandler.get_current_user = Mock(
+            return_value=User("[email protected]"))
+        response = self.get('/download_sample_info_per_prep/1')
+        self.assertEqual(response.code, 405)
+
+
 class TestDownloadUpload(TestHandlerBase):
 
     def test_download(self):

qiita_pet/webserver.py

@@ -51,7 +51,7 @@
     DownloadHandler, DownloadStudyBIOMSHandler, DownloadRelease,
     DownloadRawData, DownloadEBISampleAccessions, DownloadEBIPrepAccessions,
     DownloadUpload, DownloadPublicHandler, DownloadPublicArtifactHandler,
-    DownloadPrivateArtifactHandler)
+    DownloadSampleInfoPerPrep, DownloadPrivateArtifactHandler)
 from qiita_pet.handlers.prep_template import (
     PrepTemplateHandler, PrepTemplateGraphHandler, PrepTemplateJobHandler)
 from qiita_pet.handlers.ontology import OntologyHandler
@@ -183,6 +183,8 @@ def __init__(self):
             (r"/download_raw_data/(.*)", DownloadRawData),
             (r"/download_ebi_accessions/samples/(.*)",
                 DownloadEBISampleAccessions),
+            (r"/download_sample_info_per_prep/(.*)",
+                DownloadSampleInfoPerPrep),
             (r"/download_ebi_accessions/experiments/(.*)",
                 DownloadEBIPrepAccessions),
             (r"/download_upload/(.*)", DownloadUpload),