Merge branch 'dev' into resource-allocation-colormap

Author: Gossty

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Qiita changelog
 
+Version 2025.01
+---------------
+
+Deployed on January 15th, 2025
+
+* The Analysis owner is now displayed in the analysis list and the individual analysis page.
+* Admins can now use the per-preparation "Download Data Release" button to get a "BIOM" release; this version is focus on NPH data releases.
+* Improved complete_job creation time, which should result in Qiita jobs ([multiple steps](https://qiita.ucsd.edu/static/doc/html/dev/resource_allocation.html) finishing faster; for bencharks visit [patch 93.sql](https://github.com/qiita-spots/qiita/blob/master/qiita_db/support_files/patches/93.sql).
+* SPP improvements: TellSeq support added; plugin refactored to allow for easier additions like TellSeq in the future. Job restart greatly improved. Much improved handling of sample-names and ids that contain substrings like ‘I1’ and ‘R2’. New SequenceCount job can count sequences and base-pairs in parallel for any list of fastq files.
+* Other general fixes [#3440](https://github.com/qiita-spots/qiita/pull/3440), [#3445](https://github.com/qiita-spots/qiita/pull/3445), [#3446](https://github.com/qiita-spots/qiita/pull/3446),
+
+
 Version 2024.10
 ---------------
 
@@ -206,7 +218,7 @@ Version 2021.11
 * Allow chucked download of metadata files in analyses; this allows to process large meta-analysis (like those for The Microsetta Initiative) without worker blockage.
 * Added to the qp-qiime2 plugin the possibility of filtering tables based on system available "FeatureData[Sequence]"; to start we added 90/100/150 bps bloom tables.
 * Now we can instantiate a study via their title (Study.from_title); this will facilitate orchestration with qebil.
-* Speed up Study listing for admins and general users; the admin study display came down from 20 to 2 seconds.  
+* Speed up Study listing for admins and general users; the admin study display came down from 20 to 2 seconds.
 * Fixed the following issues: [3142](https://github.com/qiita-spots/qiita/issues/3142), [3149](https://github.com/qiita-spots/qiita/issues/3149), [3150](https://github.com/qiita-spots/qiita/issues/3150), [3119](https://github.com/qiita-spots/qiita/issues/3119), and [3160](https://github.com/qiita-spots/qiita/issues/3160).
 
 

qiita_core/__init__.py

@@ -6,4 +6,4 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-__version__ = "2024.10"
+__version__ = "2025.01"

qiita_db/__init__.py

@@ -27,7 +27,7 @@
 from . import user
 from . import processing_job
 
-__version__ = "2024.10"
+__version__ = "2025.01"
 
 __all__ = ["analysis", "artifact",  "archive", "base", "commands",
            "environment_manager", "exceptions", "investigation", "logger",

qiita_db/archive.py

@@ -116,6 +116,7 @@ def get_merging_scheme_from_job(cls, job):
             acmd = job.command
             parent = job.input_artifacts[0]
             parent_pparameters = parent.processing_parameters
+            phms = None
             if parent_pparameters is None:
                 parent_cmd_name = None
                 parent_parameters = None
@@ -125,12 +126,26 @@ def get_merging_scheme_from_job(cls, job):
                 parent_cmd_name = pcmd.name
                 parent_parameters = parent_pparameters.values
                 parent_merging_scheme = pcmd.merging_scheme
-
-            return qdb.util.human_merging_scheme(
+                if not parent_merging_scheme['ignore_parent_command']:
+                    gp = parent.parents[0]
+                    gp_params = gp.processing_parameters
+                    if gp_params is not None:
+                        gp_cmd = gp_params.command
+                        phms = qdb.util.human_merging_scheme(
+                            parent_cmd_name, parent_merging_scheme,
+                            gp_cmd.name, gp_cmd.merging_scheme,
+                            parent_parameters, [], gp_params.values)
+
+            hms = qdb.util.human_merging_scheme(
                 acmd.name, acmd.merging_scheme,
                 parent_cmd_name, parent_merging_scheme,
                 job.parameters.values, [], parent_parameters)
 
+            if phms is not None:
+                hms = qdb.util.merge_overlapping_strings(hms, phms)
+
+            return hms
+
     @classmethod
     def retrieve_feature_values(cls, archive_merging_scheme=None,
                                 features=None):

qiita_db/metadata_template/prep_template.py

@@ -808,14 +808,24 @@ def _get_node_info(workflow, node):
 
             parent_cmd_name = None
             parent_merging_scheme = None
+            phms = None
             if pcmd is not None:
                 parent_cmd_name = pcmd.name
                 parent_merging_scheme = pcmd.merging_scheme
+                if not parent_merging_scheme['ignore_parent_command']:
+                    phms = _get_node_info(workflow, parent)
 
-            return qdb.util.human_merging_scheme(
+            hms = qdb.util.human_merging_scheme(
                 ccmd.name, ccmd.merging_scheme, parent_cmd_name,
                 parent_merging_scheme, cparams, [], pparams)
 
+            # if the parent should not ignore its parent command, then we need
+            # to merge the previous result with the new one
+            if phms is not None:
+                hms = qdb.util.merge_overlapping_strings(hms, phms)
+
+            return hms
+
         def _get_predecessors(workflow, node):
             # recursive method to get predecessors of a given node
             pred = []
@@ -871,7 +881,7 @@ def _get_predecessors(workflow, node):
                          'artifact transformation']
         merging_schemes = {
             qdb.archive.Archive.get_merging_scheme_from_job(j): {
-                x: y.id for x, y in j.outputs.items()}
+                x: str(y.id) for x, y in j.outputs.items()}
             # we are going to select only the jobs that were a 'success', that
             # are not 'hidden' and that have an output - jobs that are not
             # hidden and a successs but that do not have outputs are jobs which
@@ -989,7 +999,7 @@ def _get_predecessors(workflow, node):
                     init_artifacts = {
                         wkartifact_type: f'{starting_job.id}:'}
                 else:
-                    init_artifacts = {wkartifact_type: self.artifact.id}
+                    init_artifacts = {wkartifact_type: str(self.artifact.id)}
 
             cmds_to_create.reverse()
             current_job = None

qiita_db/support_files/patches/94.sql

@@ -24,4 +24,4 @@ CREATE TABLE qiita.resource_allocation_column_names (
 INSERT INTO qiita.resource_allocation_column_names(col_name) VALUES
       ('samples'), ('columns'), ('input_size'),
       ('samples*columns'), ('samples*input_size'),
-      ('columns*input_size'), ('samples*columns*input_size');
+      ('columns*input_size'), ('samples*columns*input_size');

qiita_db/support_files/patches/test_db_sql/94.sql

@@ -0,0 +1,10 @@
+INSERT INTO qiita.allocation_equations(equation_name, expression)
+        VALUES 
+        ('mem_model1', '(k * (np.log(x))) + (x * a) + b'),
+('mem_model2', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + a'),
+('mem_model3', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.np.log(x))**3))'),
+('mem_model4', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.log(x))**2.5))'),
+('time_model1', 'a + b + ((np.log(x)) * k)'),
+('time_model2', 'a + (b * x) + ((np.log(x)) * k)'),
+('time_model3', 'a + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'),
+('time_model4', '(a * ((np.log(x))**3)) + (b * ((np.log(x))**2)) + ((np.log(x)) * k)');

qiita_db/test/test_util.py

@@ -790,9 +790,11 @@ def test_generate_analysis_list(self):
         exp = [{'mapping_files': [
                 (16, qdb.util.get_filepath_information(16)['fullpath'])],
                 'description': 'A test analysis', 'artifacts': [8, 9], 'name':
-                'SomeAnalysis', 'analysis_id': 1, 'visibility': 'private'},
+                'SomeAnalysis', 'owner': '[email protected]', 'analysis_id': 1,
+                'visibility': 'private'},
                {'mapping_files': [], 'description': 'Another test analysis',
                 'artifacts': [], 'name': 'SomeSecondAnalysis',
+                'owner': '[email protected]',
                 'analysis_id': 2, 'visibility': 'private'}]
         # removing timestamp for testing
         for i in range(len(obs)):

qiita_db/util.py

@@ -2076,7 +2076,7 @@ def generate_analysis_list(analysis_ids, public_only=False):
         return []
 
     sql = """
-        SELECT analysis_id, a.name, a.description, a.timestamp,
+        SELECT analysis_id, a.name, a.description, a.timestamp, a.email,
             array_agg(DISTINCT artifact_id),
             array_agg(DISTINCT visibility),
             array_agg(DISTINCT CASE WHEN filepath_type = 'plain_text'
@@ -2097,7 +2097,8 @@ def generate_analysis_list(analysis_ids, public_only=False):
 
         qdb.sql_connection.TRN.add(sql, [tuple(analysis_ids)])
         for row in qdb.sql_connection.TRN.execute_fetchindex():
-            aid, name, description, ts, artifacts, av, mapping_files = row
+            aid, name, description, ts, owner, artifacts, \
+                av, mapping_files = row
 
             av = 'public' if set(av) == {'public'} else 'private'
             if av != 'public' and public_only:
@@ -2118,7 +2119,7 @@ def generate_analysis_list(analysis_ids, public_only=False):
             results.append({
                 'analysis_id': aid, 'name': name, 'description': description,
                 'timestamp': ts.strftime("%m/%d/%y %H:%M:%S"),
-                'visibility': av, 'artifacts': artifacts,
+                'visibility': av, 'artifacts': artifacts, 'owner': owner,
                 'mapping_files': mapping_files})
 
     return results
@@ -2336,7 +2337,6 @@ def resource_allocation_plot(df, col_name_str, curr_column):
         Column name for the x axis that will be used to build the plots.
     curr_column: pd.Series, requirew
         Pandas Series representing a column with col_name_str.
-
     Returns
     ----------
     matplotlib.pyplot object
@@ -3008,3 +3008,25 @@ def merge_rows(rows):
                 row['node_model']]
             qdb.sql_connection.TRN.add(sql, sql_args=to_insert)
             qdb.sql_connection.TRN.execute()
+
+
+def merge_overlapping_strings(str1, str2):
+    """Helper function to merge 2 overlapping strings
+
+    Parameters
+    ----------
+    str1: str
+        Initial string
+    str2: str
+        End string
+
+    Returns
+    ----------
+    str
+        The merged strings
+    """
+    overlap = ""
+    for i in range(1, min(len(str1), len(str2)) + 1):
+        if str1.endswith(str2[:i]):
+            overlap = str2[:i]
+    return str1 + str2[len(overlap):]

qiita_pet/__init__.py

@@ -6,4 +6,4 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-__version__ = "2024.10"
+__version__ = "2025.01"

qiita_pet/handlers/analysis_handlers/base_handlers.py

@@ -85,6 +85,7 @@ def analysis_description_handler_get_request(analysis_id, user):
             'analysis_is_public': analysis.is_public,
             'analysis_description': analysis.description,
             'analysis_mapping_id': analysis.mapping_file,
+            'analysis_owner': analysis.owner.email,
             'alert_type': alert_type,
             'artifacts': artifacts,
             'analysis_reservation': analysis._slurm_reservation()[0],

qiita_pet/handlers/analysis_handlers/tests/test_base_handlers.py

@@ -38,6 +38,7 @@ def test_analysis_description_handler_get_request(self):
                'analysis_id': 1,
                'analysis_description': 'A test analysis',
                'analysis_mapping_id': 16,
+               'analysis_owner': '[email protected]',
                'analysis_is_public': False,
                'alert_type': 'info',
                'artifacts': {
@@ -67,6 +68,7 @@ def test_analysis_description_handler_get_request(self):
                'analysis_id': 1,
                'analysis_description': 'A test analysis',
                'analysis_mapping_id': 16,
+               'analysis_owner': '[email protected]',
                'analysis_is_public': False,
                'alert_type': 'info',
                'artifacts': {
@@ -98,6 +100,7 @@ def test_analysis_description_handler_get_request(self):
                'analysis_id': 1,
                'analysis_description': 'A test analysis',
                'analysis_mapping_id': 16,
+               'analysis_owner': '[email protected]',
                'analysis_is_public': False,
                'alert_type': 'danger',
                'artifacts': {

qiita_pet/handlers/api_proxy/__init__.py

@@ -38,7 +38,7 @@
 from .user import (user_jobs_get_req)
 from .util import check_access, check_fp
 
-__version__ = "2024.10"
+__version__ = "2025.01"
 
 __all__ = ['prep_template_summary_get_req', 'data_types_get_req',
            'study_get_req', 'sample_template_filepaths_get_req',