diff --git a/CHANGELOG.md b/CHANGELOG.md index c21fa43e7..ce2915f73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Qiita changelog +Version 2024.07 +--------------- + +Deployed on July 15th, 2024 + +* On June 14th, 2024 we modified the SPP to use ["fastp & minimap2 against GRCh38.p14 + Phi X 174 + T2T-CHM13v2.0, then Movi against GRCh38.p14, T2T-CHM13v2.0 + Human Pangenome Reference Consortium release 2023"](https://github.com/cguccione/human_host_filtration) to filter human-reads. +* Full refactor of the [DB patching system](https://github.com/qiita-spots/qiita/blob/master/CONTRIBUTING.md#patch-91sql) to make sure that a new production deployment has a fully empty database. +* Fully removed Qiimp from Qiita. +* Users can now add `ORCID`, `ResearchGate` and/or `GoogleScholar` information to their profile and the creation (registration) timestamp is kept in the database. Thank you @jlab. +* Admins can now track and purge non-confirmed users from the database via the GUI (`/admin/purge_users/`). Thank you @jlab. +* Added `qiita.slurm_resource_allocations` to store general job resource usage, which can be populated by `qiita_db.util.update_resource_allocation_table`. +* Added `qiita_db.util.resource_allocation_plot` to generate different models to allocate resources from a given software command based on previous jobs, thank you @Gossty ! +* The stats page map can be centered via the configuration file; additionally, the Help and Admin emails are defined also via the configuration files, thank you @jlab ! +* ``Sequel IIe``, ``Revio``, and ``Onso`` are now valid instruments for the ``PacBio_SMRT`` platform. +* Added `current_human_filtering` to the prep-information and `human_reads_filter_method` to the artifact to keep track of the method that it was used to human reads filter the raw artifact and know if it's up to date with what is expected via the best practices. +* Added `reprocess_job_id` to the prep-information so we keep track if a preparation has been reprocessed with another job. +* Other general fixes, like [#3385](https://github.com/qiita-spots/qiita/pull/3385), [#3397](https://github.com/qiita-spots/qiita/pull/3397), [#3399](https://github.com/qiita-spots/qiita/pull/3399), [#3400](https://github.com/qiita-spots/qiita/pull/3400), [#3409](https://github.com/qiita-spots/qiita/pull/3409), [#3410](https://github.com/qiita-spots/qiita/pull/3410). + + Version 2024.02 --------------- diff --git a/qiita_db/support_files/patches/test_db_sql/91.sql b/qiita_db/support_files/patches/test_db_sql/91.sql new file mode 100644 index 000000000..25d2587b2 --- /dev/null +++ b/qiita_db/support_files/patches/test_db_sql/91.sql @@ -0,0 +1,3 @@ +-- Just an empty SQL to allow the changes implemented in +-- https://github.com/qiita-spots/qiita/pull/3403 to take effect +SELECT 1; diff --git a/qiita_db/util.py b/qiita_db/util.py index 7ca5e4d24..e77648cb0 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -49,13 +49,13 @@ from bcrypt import hashpw, gensalt from functools import partial from os.path import join, basename, isdir, exists, getsize -from os import walk, remove, listdir, rename, stat +from os import walk, remove, listdir, rename, stat, makedirs from glob import glob from shutil import move, rmtree, copy as shutil_copy from openpyxl import load_workbook from tempfile import mkstemp from csv import writer as csv_writer -from datetime import datetime +from datetime import datetime, timedelta from time import time as now from itertools import chain from contextlib import contextmanager @@ -64,18 +64,15 @@ import hashlib from smtplib import SMTP, SMTP_SSL, SMTPException -from os import makedirs from errno import EEXIST from qiita_core.exceptions import IncompetentQiitaDeveloperError from qiita_core.qiita_settings import qiita_config from subprocess import check_output import qiita_db as qdb - from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from datetime import timedelta import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -2730,19 +2727,19 @@ def update_resource_allocation_table(weeks=1, test=None): dates = ['', ''] + slurm_external_id = 0 + start_date = datetime.strptime('2023-04-28', '%Y-%m-%d') with qdb.sql_connection.TRN: sql = sql_timestamp qdb.sql_connection.TRN.add(sql) res = qdb.sql_connection.TRN.execute_fetchindex() - slurm_external_id, timestamp = res[0] - if slurm_external_id is None: - slurm_external_id = 0 - if timestamp is None: - dates[0] = datetime.strptime('2023-04-28', '%Y-%m-%d') - else: - dates[0] = timestamp - date1 = dates[0] + timedelta(weeks) - dates[1] = date1.strftime('%Y-%m-%d') + if res: + sei, sd = res[0] + if sei is not None: + slurm_external_id = sei + if sd is not None: + start_date = sd + dates = [start_date, start_date + timedelta(weeks=weeks)] sql_command = """ SELECT @@ -2769,27 +2766,23 @@ def update_resource_allocation_table(weeks=1, test=None): """ df = pd.DataFrame() with qdb.sql_connection.TRN: - sql = sql_command - qdb.sql_connection.TRN.add(sql, sql_args=[slurm_external_id]) + qdb.sql_connection.TRN.add(sql_command, sql_args=[slurm_external_id]) res = qdb.sql_connection.TRN.execute_fetchindex() - columns = ["processing_job_id", 'external_id'] - df = pd.DataFrame(res, columns=columns) + df = pd.DataFrame(res, columns=["processing_job_id", 'external_id']) df['external_id'] = df['external_id'].astype(int) data = [] sacct = [ 'sacct', '-p', '--format=JobID,ElapsedRaw,MaxRSS,Submit,Start,End,CPUTimeRAW,' - 'ReqMem,AllocCPUs,AveVMSize', '--starttime', dates[0], '--endtime', - dates[1], '--user', 'qiita', '--state', 'CD'] + 'ReqMem,AllocCPUs,AveVMSize', '--starttime', + dates[0].strftime('%Y-%m-%d'), '--endtime', + dates[1].strftime('%Y-%m-%d'), '--user', 'qiita', '--state', 'CD'] if test is not None: slurm_data = test else: - try: - rvals = StringIO(check_output(sacct)).decode('ascii') - except TypeError as e: - raise e + rvals = check_output(sacct).decode('ascii') slurm_data = pd.read_csv(StringIO(rvals), sep='|') # In slurm, each JobID is represented by 3 rows in the dataframe: diff --git a/qiita_pet/handlers/user_handlers.py b/qiita_pet/handlers/user_handlers.py index c85677cb8..1783fba37 100644 --- a/qiita_pet/handlers/user_handlers.py +++ b/qiita_pet/handlers/user_handlers.py @@ -187,11 +187,11 @@ def validator_rgate_id(form, field): "Receive Processing Job Emails?") social_orcid = StringField( - "ORCID", [validator_orcid_id], description="0000-0002-0975-9019") + "ORCID", [validator_orcid_id], description="") social_googlescholar = StringField( - "Google Scholar", [validator_gscholar_id], description="_e3QL94AAAAJ") + "Google Scholar", [validator_gscholar_id], description="") social_researchgate = StringField( - "ResearchGate", [validator_rgate_id], description="Rob-Knight") + "ResearchGate", [validator_rgate_id], description="") class UserProfileHandler(BaseHandler): diff --git a/qiita_pet/support_files/doc/source/gettingstartedguide/index.rst b/qiita_pet/support_files/doc/source/gettingstartedguide/index.rst index afb081e7d..b608d5e7e 100755 --- a/qiita_pet/support_files/doc/source/gettingstartedguide/index.rst +++ b/qiita_pet/support_files/doc/source/gettingstartedguide/index.rst @@ -76,11 +76,6 @@ Editing a Study .. _prepare-information-files: -Creating and Working With Sample information --------------------------------------------- - -* The :doc:`../qiimp` tab located at the top of the page is a useful tool for creating Qiita and EBI compatible metadata spreadsheets. - Example files ~~~~~~~~~~~~~