From 57d1b146729223da9daa286a1cc75db6f2923746 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 29 Jan 2024 12:35:46 -0700 Subject: [PATCH 1/4] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1756c7238..24ad9a78a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ Deployed on January 8th, 2024 * Workflow definitions can now use sample or preparation information columns/values to differentiate between them. * Updated the Adapter and host filtering plugin (qp-fastp-minimap2) to v2023.12 addressing a bug in adapter filtering; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/qp-fastp-minimap2.html). * Other fixes: [3334](https://github.com/qiita-spots/qiita/pull/3334), [3338](https://github.com/qiita-spots/qiita/pull/3338). Thank you @sjanssen2. -* The internal Sequence Processing Pipeline is now using the human pan-genome reference, together with the GRCh38 genome + PhiX and CHM13 genome for human host filtering. +* The internal Sequence Processing Pipeline is now using the human pan-genome reference, together with the GRCh38 genome + PhiX and T2T-CHM13v2.0 genome for human host filtering. Version 2023.10 From 346daa4e7f8b421aa9cc44747632fb53bb33736c Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Wed, 3 Jul 2024 14:40:06 -0600 Subject: [PATCH 2/4] general fixes after deployment --- CHANGELOG.md | 18 ++++++++++++++++++ .../support_files/patches/test_db_sql/91.sql | 3 +++ qiita_db/util.py | 18 +++++++++--------- qiita_pet/handlers/user_handlers.py | 6 +++--- .../doc/source/gettingstartedguide/index.rst | 5 ----- 5 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 qiita_db/support_files/patches/test_db_sql/91.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index c21fa43e7..9e601d2a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Qiita changelog +Version 2024.02 +--------------- + +Deployed on July 15th, 2024 + +* Full refactor of the [DB patching system](https://github.com/qiita-spots/qiita/blob/master/CONTRIBUTING.md#patch-91sql) to make sure that a new production deployment has a fully empty database. +* Fully removed Qiimp from Qiita. +* Users can now add `ORCID`, `ResearchGate` and/or `GoogleScholar` information to their profile and the creation (registration) timestamp is kept in the database. Thank you @jlab. +* Admins can now track and purge non-confirmed users from the database via the GUI (`/admin/purge_users/`). Thank you @jlab. +* Added `qiita.slurm_resource_allocations` to store general job resource usage, which can be populated by `qiita_db.util.update_resource_allocation_table`. +* Added `qiita_db.util.resource_allocation_plot` to generate different models to allocate resources from a given software command based on previous jobs, thank you @Gossty ! +* The stats page map can be centered via the configuration file; additionally, the Help and Admin emails are defined also via the configuration files, thank you @jlab ! +* ``Sequel IIe``, ``Revio``, and ``Onso`` are now valid instruments for the ``PacBio_SMRT`` platform. +* Added `current_human_filtering` to the prep-information and `human_reads_filter_method` to the artifact to keep track of the method that it was used to human reads filter the raw artifact and know if it's up to date with what is expected via the best practices. +* Added `reprocess_job_id` to the prep-information so we keep track if a preparation has been reprocessed with another job. +* Other general fixes, like [#3385](https://github.com/qiita-spots/qiita/pull/3385), [#3397](https://github.com/qiita-spots/qiita/pull/3397), [#3399](https://github.com/qiita-spots/qiita/pull/3399), [#3400](https://github.com/qiita-spots/qiita/pull/3400), [#3409](https://github.com/qiita-spots/qiita/pull/3409), [#3410](https://github.com/qiita-spots/qiita/pull/3410). + + Version 2024.02 --------------- diff --git a/qiita_db/support_files/patches/test_db_sql/91.sql b/qiita_db/support_files/patches/test_db_sql/91.sql new file mode 100644 index 000000000..25d2587b2 --- /dev/null +++ b/qiita_db/support_files/patches/test_db_sql/91.sql @@ -0,0 +1,3 @@ +-- Just an empty SQL to allow the changes implemented in +-- https://github.com/qiita-spots/qiita/pull/3403 to take effect +SELECT 1; diff --git a/qiita_db/util.py b/qiita_db/util.py index 7ca5e4d24..8f3145cd2 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2730,19 +2730,19 @@ def update_resource_allocation_table(weeks=1, test=None): dates = ['', ''] + slurm_external_id = 0 + start_date = datetime.strptime('2023-04-28', '%Y-%m-%d') with qdb.sql_connection.TRN: sql = sql_timestamp qdb.sql_connection.TRN.add(sql) res = qdb.sql_connection.TRN.execute_fetchindex() - slurm_external_id, timestamp = res[0] - if slurm_external_id is None: - slurm_external_id = 0 - if timestamp is None: - dates[0] = datetime.strptime('2023-04-28', '%Y-%m-%d') - else: - dates[0] = timestamp - date1 = dates[0] + timedelta(weeks) - dates[1] = date1.strftime('%Y-%m-%d') + if res: + sei, sd = res[0] + if sei is not None: + slurm_external_id = sei + if sd is not None: + start_date = sd + dates = [start_date, start_date + timedelta(weeks)] sql_command = """ SELECT diff --git a/qiita_pet/handlers/user_handlers.py b/qiita_pet/handlers/user_handlers.py index c85677cb8..1783fba37 100644 --- a/qiita_pet/handlers/user_handlers.py +++ b/qiita_pet/handlers/user_handlers.py @@ -187,11 +187,11 @@ def validator_rgate_id(form, field): "Receive Processing Job Emails?") social_orcid = StringField( - "ORCID", [validator_orcid_id], description="0000-0002-0975-9019") + "ORCID", [validator_orcid_id], description="") social_googlescholar = StringField( - "Google Scholar", [validator_gscholar_id], description="_e3QL94AAAAJ") + "Google Scholar", [validator_gscholar_id], description="") social_researchgate = StringField( - "ResearchGate", [validator_rgate_id], description="Rob-Knight") + "ResearchGate", [validator_rgate_id], description="") class UserProfileHandler(BaseHandler): diff --git a/qiita_pet/support_files/doc/source/gettingstartedguide/index.rst b/qiita_pet/support_files/doc/source/gettingstartedguide/index.rst index afb081e7d..b608d5e7e 100755 --- a/qiita_pet/support_files/doc/source/gettingstartedguide/index.rst +++ b/qiita_pet/support_files/doc/source/gettingstartedguide/index.rst @@ -76,11 +76,6 @@ Editing a Study .. _prepare-information-files: -Creating and Working With Sample information --------------------------------------------- - -* The :doc:`../qiimp` tab located at the top of the page is a useful tool for creating Qiita and EBI compatible metadata spreadsheets. - Example files ~~~~~~~~~~~~~ From 5886d11a18a40eec241878472343f52499bee4ad Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Wed, 3 Jul 2024 15:25:42 -0600 Subject: [PATCH 3/4] some updates --- CHANGELOG.md | 3 ++- qiita_db/util.py | 14 ++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e601d2a0..5595bccc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Qiita changelog -Version 2024.02 +Version 2024.07 --------------- Deployed on July 15th, 2024 @@ -16,6 +16,7 @@ Deployed on July 15th, 2024 * Added `current_human_filtering` to the prep-information and `human_reads_filter_method` to the artifact to keep track of the method that it was used to human reads filter the raw artifact and know if it's up to date with what is expected via the best practices. * Added `reprocess_job_id` to the prep-information so we keep track if a preparation has been reprocessed with another job. * Other general fixes, like [#3385](https://github.com/qiita-spots/qiita/pull/3385), [#3397](https://github.com/qiita-spots/qiita/pull/3397), [#3399](https://github.com/qiita-spots/qiita/pull/3399), [#3400](https://github.com/qiita-spots/qiita/pull/3400), [#3409](https://github.com/qiita-spots/qiita/pull/3409), [#3410](https://github.com/qiita-spots/qiita/pull/3410). +* On June 14th, 2024 we modified the SPP to use XXXX to filter human-reads from the per-sample-FASTQ loaded to Qiita. Version 2024.02 diff --git a/qiita_db/util.py b/qiita_db/util.py index 8f3145cd2..c276a3bd9 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -49,13 +49,13 @@ from bcrypt import hashpw, gensalt from functools import partial from os.path import join, basename, isdir, exists, getsize -from os import walk, remove, listdir, rename, stat +from os import walk, remove, listdir, rename, stat, makedirs from glob import glob from shutil import move, rmtree, copy as shutil_copy from openpyxl import load_workbook from tempfile import mkstemp from csv import writer as csv_writer -from datetime import datetime +from datetime import datetime, timedelta from time import time as now from itertools import chain from contextlib import contextmanager @@ -64,18 +64,15 @@ import hashlib from smtplib import SMTP, SMTP_SSL, SMTPException -from os import makedirs from errno import EEXIST from qiita_core.exceptions import IncompetentQiitaDeveloperError from qiita_core.qiita_settings import qiita_config from subprocess import check_output import qiita_db as qdb - from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from datetime import timedelta import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -2742,7 +2739,7 @@ def update_resource_allocation_table(weeks=1, test=None): slurm_external_id = sei if sd is not None: start_date = sd - dates = [start_date, start_date + timedelta(weeks)] + dates = [start_date, start_date + timedelta(weeks=weeks)] sql_command = """ SELECT @@ -2780,8 +2777,9 @@ def update_resource_allocation_table(weeks=1, test=None): sacct = [ 'sacct', '-p', '--format=JobID,ElapsedRaw,MaxRSS,Submit,Start,End,CPUTimeRAW,' - 'ReqMem,AllocCPUs,AveVMSize', '--starttime', dates[0], '--endtime', - dates[1], '--user', 'qiita', '--state', 'CD'] + 'ReqMem,AllocCPUs,AveVMSize', '--starttime', + dates[0].strftime('%Y-%m-%d'), '--endtime', + dates[1].strftime('%Y-%m-%d'), '--user', 'qiita', '--state', 'CD'] if test is not None: slurm_data = test From c29bb0648d21f6241fcf225c5ab9b1ef056654e8 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 5 Jul 2024 11:53:07 -0600 Subject: [PATCH 4/4] some minor changes --- CHANGELOG.md | 2 +- qiita_db/util.py | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5595bccc1..ce2915f73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Version 2024.07 Deployed on July 15th, 2024 +* On June 14th, 2024 we modified the SPP to use ["fastp & minimap2 against GRCh38.p14 + Phi X 174 + T2T-CHM13v2.0, then Movi against GRCh38.p14, T2T-CHM13v2.0 + Human Pangenome Reference Consortium release 2023"](https://github.com/cguccione/human_host_filtration) to filter human-reads. * Full refactor of the [DB patching system](https://github.com/qiita-spots/qiita/blob/master/CONTRIBUTING.md#patch-91sql) to make sure that a new production deployment has a fully empty database. * Fully removed Qiimp from Qiita. * Users can now add `ORCID`, `ResearchGate` and/or `GoogleScholar` information to their profile and the creation (registration) timestamp is kept in the database. Thank you @jlab. @@ -16,7 +17,6 @@ Deployed on July 15th, 2024 * Added `current_human_filtering` to the prep-information and `human_reads_filter_method` to the artifact to keep track of the method that it was used to human reads filter the raw artifact and know if it's up to date with what is expected via the best practices. * Added `reprocess_job_id` to the prep-information so we keep track if a preparation has been reprocessed with another job. * Other general fixes, like [#3385](https://github.com/qiita-spots/qiita/pull/3385), [#3397](https://github.com/qiita-spots/qiita/pull/3397), [#3399](https://github.com/qiita-spots/qiita/pull/3399), [#3400](https://github.com/qiita-spots/qiita/pull/3400), [#3409](https://github.com/qiita-spots/qiita/pull/3409), [#3410](https://github.com/qiita-spots/qiita/pull/3410). -* On June 14th, 2024 we modified the SPP to use XXXX to filter human-reads from the per-sample-FASTQ loaded to Qiita. Version 2024.02 diff --git a/qiita_db/util.py b/qiita_db/util.py index c276a3bd9..e77648cb0 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2766,11 +2766,9 @@ def update_resource_allocation_table(weeks=1, test=None): """ df = pd.DataFrame() with qdb.sql_connection.TRN: - sql = sql_command - qdb.sql_connection.TRN.add(sql, sql_args=[slurm_external_id]) + qdb.sql_connection.TRN.add(sql_command, sql_args=[slurm_external_id]) res = qdb.sql_connection.TRN.execute_fetchindex() - columns = ["processing_job_id", 'external_id'] - df = pd.DataFrame(res, columns=columns) + df = pd.DataFrame(res, columns=["processing_job_id", 'external_id']) df['external_id'] = df['external_id'].astype(int) data = [] @@ -2784,10 +2782,7 @@ def update_resource_allocation_table(weeks=1, test=None): if test is not None: slurm_data = test else: - try: - rvals = StringIO(check_output(sacct)).decode('ascii') - except TypeError as e: - raise e + rvals = check_output(sacct).decode('ascii') slurm_data = pd.read_csv(StringIO(rvals), sep='|') # In slurm, each JobID is represented by 3 rows in the dataframe: