From 059f016486d05f9604b39851aeff7cd2248feab9 Mon Sep 17 00:00:00 2001 From: Keita Urashima Date: Thu, 4 Dec 2025 09:10:32 +0900 Subject: [PATCH 1/5] Update submission from DDBJ Record --- app/controllers/application_controller.rb | 1 + .../submission_requests_controller.rb | 32 ++ .../submission_updates_controller.rb | 23 ++ app/controllers/submissions_controller.rb | 70 +---- .../apply_submission_request_job.rb} | 52 ++-- app/jobs/apply_submission_update_job.rb | 67 ++++ app/jobs/submit_job.rb | 2 +- app/jobs/validate_job.rb | 6 +- app/jobs/validate_submission_request_job.rb | 5 + app/models/accession_renewal.rb | 10 - .../accession_renewal_validation_detail.rb | 3 - app/models/bioproject_submission_param.rb | 11 - app/models/concerns/validation_subject.rb | 36 +++ app/models/database.rb | 12 - app/models/database/bioproject.rb | 5 - .../database/bioproject/file_validator.rb | 17 - app/models/database/bioproject/submitter.rb | 292 ------------------ app/models/database/biosample.rb | 5 - .../database/biosample/file_validator.rb | 40 --- app/models/database/biosample/submitter.rb | 291 ----------------- app/models/database/dra.rb | 5 - app/models/database/dra/file_validator.rb | 48 --- app/models/database/dra/submitter.rb | 5 - app/models/database/gea.rb | 5 - app/models/database/gea/file_validator.rb | 44 --- app/models/database/gea/submitter.rb | 5 - app/models/database/jvar.rb | 5 - app/models/database/jvar/file_validator.rb | 5 - app/models/database/jvar/submitter.rb | 5 - app/models/database/metabobank.rb | 5 - .../database/metabobank/file_validator.rb | 55 ---- app/models/database/metabobank/submitter.rb | 5 - app/models/database/trad.rb | 5 - app/models/database/trad/file_validator.rb | 108 ------- app/models/database/trad2.rb | 5 - app/models/database/trad2/file_validator.rb | 37 --- app/models/database/trad2/submitter.rb | 5 - .../trad => }/ddbj_record_validator.rb | 73 ++++- app/models/obj.rb | 59 ---- app/models/submission.rb | 30 +- app/models/submission_request.rb | 8 + app/models/submission_update.rb | 7 + app/models/user.rb | 5 +- app/models/validation.rb | 81 +---- app/models/validation_detail.rb | 4 +- app/views/application/_blob.json.jb | 4 + .../_submission_request.json.jb | 12 + app/views/submission_requests/index.json.jb | 1 + app/views/submission_requests/show.json.jb | 1 + app/views/submission_updates/show.json.jb | 12 + app/views/submissions/_submission.json.jb | 42 ++- app/views/submissions/index.json.jb | 2 +- app/views/submissions/show.json.jb | 2 +- app/views/validations/_validation.json.jb | 37 +-- config/initializers/active_storage.rb | 6 +- config/initializers/cors.rb | 5 + config/routes.rb | 17 +- ...251119073632_create_submission_requests.rb | 58 ++++ db/schema.rb | 91 ++---- spec/factories/submission_requests.rb | 9 + spec/factories/validations.rb | 18 -- spec/requests/submission_requests_spec.rb | 73 +++++ spec/requests/submissions_spec.rb | 169 +--------- spec/requests/validations_spec.rb | 9 +- web/app/config/environment.ts | 1 + web/app/router.ts | 15 +- web/app/routes/index.ts | 2 +- web/app/routes/request.gts | 14 + web/app/routes/requests.ts | 15 + web/app/routes/requests/index.ts | 16 + web/app/routes/update.ts | 14 + web/app/templates/application.gts | 6 +- web/app/templates/request.gts | 111 +++++++ web/app/templates/requests/index.gts | 34 ++ web/app/templates/requests/new.gts | 75 +++++ web/app/templates/submission/index.gts | 214 +++---------- web/app/templates/update.gts | 76 +++++ web/app/templates/updates/new.gts | 75 +++++ web/config/environment.js | 2 + web/package.json | 3 +- web/pnpm-lock.yaml | 48 ++- 81 files changed, 1129 insertions(+), 1759 deletions(-) create mode 100644 app/controllers/submission_requests_controller.rb create mode 100644 app/controllers/submission_updates_controller.rb rename app/{models/database/trad/submitter.rb => jobs/apply_submission_request_job.rb} (50%) create mode 100644 app/jobs/apply_submission_update_job.rb create mode 100644 app/jobs/validate_submission_request_job.rb delete mode 100644 app/models/accession_renewal.rb delete mode 100644 app/models/accession_renewal_validation_detail.rb delete mode 100644 app/models/bioproject_submission_param.rb create mode 100644 app/models/concerns/validation_subject.rb delete mode 100644 app/models/database.rb delete mode 100644 app/models/database/bioproject.rb delete mode 100644 app/models/database/bioproject/file_validator.rb delete mode 100644 app/models/database/bioproject/submitter.rb delete mode 100644 app/models/database/biosample.rb delete mode 100644 app/models/database/biosample/file_validator.rb delete mode 100644 app/models/database/biosample/submitter.rb delete mode 100644 app/models/database/dra.rb delete mode 100644 app/models/database/dra/file_validator.rb delete mode 100644 app/models/database/dra/submitter.rb delete mode 100644 app/models/database/gea.rb delete mode 100644 app/models/database/gea/file_validator.rb delete mode 100644 app/models/database/gea/submitter.rb delete mode 100644 app/models/database/jvar.rb delete mode 100644 app/models/database/jvar/file_validator.rb delete mode 100644 app/models/database/jvar/submitter.rb delete mode 100644 app/models/database/metabobank.rb delete mode 100644 app/models/database/metabobank/file_validator.rb delete mode 100644 app/models/database/metabobank/submitter.rb delete mode 100644 app/models/database/trad.rb delete mode 100644 app/models/database/trad/file_validator.rb delete mode 100644 app/models/database/trad2.rb delete mode 100644 app/models/database/trad2/file_validator.rb delete mode 100644 app/models/database/trad2/submitter.rb rename app/models/{database/trad => }/ddbj_record_validator.rb (65%) delete mode 100644 app/models/obj.rb create mode 100644 app/models/submission_request.rb create mode 100644 app/models/submission_update.rb create mode 100644 app/views/application/_blob.json.jb create mode 100644 app/views/submission_requests/_submission_request.json.jb create mode 100644 app/views/submission_requests/index.json.jb create mode 100644 app/views/submission_requests/show.json.jb create mode 100644 app/views/submission_updates/show.json.jb create mode 100644 db/migrate/20251119073632_create_submission_requests.rb create mode 100644 spec/factories/submission_requests.rb create mode 100644 spec/requests/submission_requests_spec.rb create mode 100644 web/app/routes/request.gts create mode 100644 web/app/routes/requests.ts create mode 100644 web/app/routes/requests/index.ts create mode 100644 web/app/routes/update.ts create mode 100644 web/app/templates/request.gts create mode 100644 web/app/templates/requests/index.gts create mode 100644 web/app/templates/requests/new.gts create mode 100644 web/app/templates/update.gts create mode 100644 web/app/templates/updates/new.gts diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 2d00fb829..ccaa57acc 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,5 +1,6 @@ class ApplicationController < ActionController::API include ActionController::HttpAuthentication::Token::ControllerMethods + include Pagy::Method before_action :authenticate! diff --git a/app/controllers/submission_requests_controller.rb b/app/controllers/submission_requests_controller.rb new file mode 100644 index 000000000..b0a13c15f --- /dev/null +++ b/app/controllers/submission_requests_controller.rb @@ -0,0 +1,32 @@ +class SubmissionRequestsController < ApplicationController + def index + requests = current_user.submission_requests.includes( + :submission, + validation_with_validity: :details, + ) + + pagy, @requests = pagy(requests.order(id: :desc)) + + response.headers.merge! pagy.headers_hash + end + + def show + @request = current_user.submission_requests.find(params[:id]) + end + + def create + @request = current_user.submission_requests.create!(request_params) + + ValidateSubmissionRequestJob.perform_later @request + + render :show, status: :accepted + end + + private + + def request_params + params.expect(submission_request: [ + :ddbj_record + ]) + end +end diff --git a/app/controllers/submission_updates_controller.rb b/app/controllers/submission_updates_controller.rb new file mode 100644 index 000000000..9c6f0f57f --- /dev/null +++ b/app/controllers/submission_updates_controller.rb @@ -0,0 +1,23 @@ +class SubmissionUpdatesController < ApplicationController + def show + @update = current_user.submission_updates.find(params[:id]) + end + + def create + submission = current_user.submissions.find(params[:submission_id]) + + @update = submission.updates.create!(update_params) + + ApplySubmissionUpdateJob.perform_later @update + + render :show, status: :accepted + end + + private + + def update_params + params.expect(submission_update: [ + :ddbj_record + ]) + end +end diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 0fcaf0d90..fab7141a2 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -1,65 +1,23 @@ class SubmissionsController < ApplicationController - include Pagy::Method - - def index - submissions = search_submissions.order(id: :desc) - - pagy, @submissions = pagy(submissions, page: params[:page]) - - response.headers.merge! pagy.headers_hash - rescue Pagy::OverflowError => e - render json: { - error: e.message - }, status: :bad_request - end - def show - @submission = user_submissions.includes(accessions: :renewals).find(params.expect(:id)) + @submission = current_user.submissions.includes( + :updates + ).order( + 'submission_updates.id DESC' + ).find(params.expect(:id)) end def create - validation = current_user.validations.find(params.require(:validation_id)) - param = Database::MAPPING.fetch(validation.db).build_param(params) - @submission = Submission.create!(**submission_params, param:) - - SubmitJob.perform_later @submission - - render :show, status: :created - end - - private - - def submission_params - params.permit(:validation_id, :visibility) - end - - def user_submissions - current_user.submissions.includes( - validation: [ - :user, - - objs: [ - :file_blob, - :validation_details - ] - ] - ) - end - - def search_submissions - db, created_at_after, created_at_before, result = params.values_at( - :db, - :created_at_after, - :created_at_before, - :result - ).map(&:presence) - - submissions = user_submissions + request = current_user.submission_requests.valid_only.joins( + :validation + ).where( + validations: { + finished_at: 1.day.ago.. + } + ).find(params[:submission_request_id]) - submissions = submissions.where(validations: {db: db.split(',')}) if db - submissions = submissions.where(created_at: created_at_after..created_at_before) if created_at_after || created_at_before - submissions = submissions.where(result: result.split(',')) if result + ApplySubmissionRequestJob.perform_later request - submissions + head :accepted end end diff --git a/app/models/database/trad/submitter.rb b/app/jobs/apply_submission_request_job.rb similarity index 50% rename from app/models/database/trad/submitter.rb rename to app/jobs/apply_submission_request_job.rb index 8e19818aa..554b418c5 100644 --- a/app/models/database/trad/submitter.rb +++ b/app/jobs/apply_submission_request_job.rb @@ -1,11 +1,29 @@ -class Database::Trad::Submitter - def submit(submission) - validation = submission.validation +class ApplySubmissionRequestJob < ApplicationJob + def perform(request) + ActiveRecord::Base.transaction do + request.applying! + request.create_submission! + end + + begin + apply request + rescue => e + Rails.error.report e + + request.update!( + status: :submission_failed, + error_message: e.message + ) + else + request.applied! + request.validation.write_submission_files to: submission.dir + end + end - return unless validation.via_ddbj_record? + private - obj = validation.objs.find_sole_by(_id: 'DDBJRecord') - record = JSON.parse(obj.file.download, symbolize_names: true) + def apply(request) + record = JSON.parse(request.ddbj_record.download, symbolize_names: true) entries = record.dig(:sequence, :entries) aa_count, na_count = entries.partition { aa?(it) }.map(&:size) @@ -14,13 +32,13 @@ def submit(submission) na_nums = Sequence.allocate!(:jpo_na, na_count) aa_nums = Sequence.allocate!(:jpo_aa, aa_count) - entry_id_to_attrs = submission.accessions.insert_all(entries.map {|entry| + entry_id_to_attrs = request.submission.accessions.insert_all(entries.map {|entry| { number: (aa?(entry) ? aa_nums : na_nums).shift, entry_id: entry[:id] } }, **{ - unique_by: %i[number entry_id version], + unique_by: :number, returning: %i[entry_id number version last_updated_at] }).index_by { it['entry_id'] @@ -37,19 +55,13 @@ def submit(submission) ) end - filename = obj.file.filename - - validation.objs.create!( - _id: 'DDBJRecord', - validity: :valid, - destination: obj.destination, + filename = request.ddbj_record.filename - file: { - io: StringIO.new(JSON.pretty_generate(record)), - filename: "#{filename.base}-#{Time.current.iso8601}.#{filename.extension}", - content_type: obj.file.content_type - } - ) + request.submission.update! ddbj_record: { + io: StringIO.new(JSON.pretty_generate(record)), + filename: "#{filename.base}-submitted.#{filename.extension}", + content_type: request.ddbj_record.content_type + } end end diff --git a/app/jobs/apply_submission_update_job.rb b/app/jobs/apply_submission_update_job.rb new file mode 100644 index 000000000..46993a0ab --- /dev/null +++ b/app/jobs/apply_submission_update_job.rb @@ -0,0 +1,67 @@ +class ApplySubmissionUpdateJob < ApplicationJob + def perform(update) + DDBJRecordValidator.validate update + + return unless update.ready_to_apply? + + begin + update.applying! + + apply update + rescue => e + Rails.error.report e + + update.update!( + status: :application_failed, + error_message: e.message + ) + else + update.applied! + end + end + + private + + def apply(update) + record = JSON.parse(update.ddbj_record.download, symbolize_names: true) + entries = record.dig(:sequence, :entries) + + ActiveRecord::Base.transaction do + number_to_accession = update.submission.accessions.index_by(&:number) + now = Time.current + + accession_to_attrs = update.submission.accessions.upsert_all(entries.map {|entry| + acc = number_to_accession.fetch(entry[:accession]) + + { + **acc.attributes, + entry_id: entry[:id], + version: acc.version.succ, + last_updated_at: now + } + }, **{ + unique_by: :number, + returning: %i[number version last_updated_at] + }).index_by { + it['number'] + }.transform_values(&:deep_symbolize_keys) + + entries.each do |entry| + next unless attrs = accession_to_attrs[entry[:accession]] + + attrs => {version:, last_updated_at:} + + entry.update( + version:, + last_updated: Time.zone.parse(last_updated_at).iso8601 + ) + end + + update.submission.update! ddbj_record: { + io: StringIO.new(JSON.pretty_generate(record)), + filename: update.ddbj_record.filename, + content_type: update.ddbj_record.content_type + } + end + end +end diff --git a/app/jobs/submit_job.rb b/app/jobs/submit_job.rb index 0e92a96e0..0389336eb 100644 --- a/app/jobs/submit_job.rb +++ b/app/jobs/submit_job.rb @@ -1,6 +1,6 @@ class SubmitJob < ApplicationJob def perform(submission) - submission.update! progress: :running, started_at: Time.current + submission.update! progress: :running submitter = Database::MAPPING.fetch(submission.validation.db)::Submitter.new diff --git a/app/jobs/validate_job.rb b/app/jobs/validate_job.rb index 0ff0b4b67..ffb053f59 100644 --- a/app/jobs/validate_job.rb +++ b/app/jobs/validate_job.rb @@ -1,6 +1,6 @@ class ValidateJob < ApplicationJob def perform(validation) - validation.update! progress: :running, started_at: Time.current + validation.update! progress: :running ActiveRecord::Base.transaction do validator = "Database::#{validation.db}::#{validation.via.camelize}Validator".constantize.new @@ -10,14 +10,10 @@ def perform(validation) rescue => e Rails.error.report e - validation.objs.base.validity_error! - validation.objs.base.validation_details.create!( severity: 'error', message: e.message ) - else - validation.objs.base.validity_valid! unless validation.objs.base.validity end raise ActiveRecord::Rollback if validation.reload.canceled? diff --git a/app/jobs/validate_submission_request_job.rb b/app/jobs/validate_submission_request_job.rb new file mode 100644 index 000000000..ab2c4e116 --- /dev/null +++ b/app/jobs/validate_submission_request_job.rb @@ -0,0 +1,5 @@ +class ValidateSubmissionRequestJob < ApplicationJob + def perform(request) + DDBJRecordValidator.validate request + end +end diff --git a/app/models/accession_renewal.rb b/app/models/accession_renewal.rb deleted file mode 100644 index a0b77b56a..000000000 --- a/app/models/accession_renewal.rb +++ /dev/null @@ -1,10 +0,0 @@ -class AccessionRenewal < ApplicationRecord - belongs_to :accession, inverse_of: :renewals - - has_many :validation_details, dependent: :destroy, class_name: 'AccessionRenewalValidationDetail', inverse_of: :renewal - - has_one_attached :file - - enum :progress, %w[waiting running finished canceled].index_by(&:to_sym) - enum :validity, %w[valid invalid error].index_by(&:to_sym), prefix: true -end diff --git a/app/models/accession_renewal_validation_detail.rb b/app/models/accession_renewal_validation_detail.rb deleted file mode 100644 index 1c74c79ed..000000000 --- a/app/models/accession_renewal_validation_detail.rb +++ /dev/null @@ -1,3 +0,0 @@ -class AccessionRenewalValidationDetail < ApplicationRecord - belongs_to :renewal, class_name: 'AccessionRenewal' -end diff --git a/app/models/bioproject_submission_param.rb b/app/models/bioproject_submission_param.rb deleted file mode 100644 index 3d42a5d72..000000000 --- a/app/models/bioproject_submission_param.rb +++ /dev/null @@ -1,11 +0,0 @@ -class BioProjectSubmissionParam < ApplicationRecord - has_one :submission, as: :param, touch: true - - validates :umbrella, inclusion: {in: [true, false]} - - def as_json - { - umbrella: - } - end -end diff --git a/app/models/concerns/validation_subject.rb b/app/models/concerns/validation_subject.rb new file mode 100644 index 000000000..10e6660c7 --- /dev/null +++ b/app/models/concerns/validation_subject.rb @@ -0,0 +1,36 @@ +module ValidationSubject + extend ActiveSupport::Concern + + included do + has_one :validation, dependent: :destroy, as: :subject + has_one :validation_with_validity, -> { with_validity }, class_name: 'Validation', as: :subject + + enum :status, { + waiting: 0, + validating: 1, + validation_failed: 2, + ready_to_apply: 3, + applying: 4, + applied: 5, + application_failed: 6 + } + + scope :with_validity, -> { + left_joins( + validation: :details + ).group( + "#{quoted_table_name}.id" + ).select("#{quoted_table_name}.*", <<~SQL) + CASE + WHEN validations.progress != 'finished' THEN NULL + WHEN COUNT(CASE WHEN validation_details.severity = 'error' THEN 1 END) = 0 THEN 'valid' + ELSE 'invalid' + END AS validity + SQL + } + + scope :valid_only, -> { + with_validity.having("validity = 'valid'") + } + end +end diff --git a/app/models/database.rb b/app/models/database.rb deleted file mode 100644 index 5c863a754..000000000 --- a/app/models/database.rb +++ /dev/null @@ -1,12 +0,0 @@ -module Database - MAPPING = { - 'BioProject' => BioProject, - 'BioSample' => BioSample, - 'DRA' => DRA, - 'GEA' => GEA, - 'JVar' => JVar, - 'MetaboBank' => MetaboBank, - 'Trad2' => Trad2, - 'Trad' => Trad - } -end diff --git a/app/models/database/bioproject.rb b/app/models/database/bioproject.rb deleted file mode 100644 index 0c5791285..000000000 --- a/app/models/database/bioproject.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Database::BioProject - def self.build_param(params) - BioProjectSubmissionParam.new(params.permit(:umbrella)) - end -end diff --git a/app/models/database/bioproject/file_validator.rb b/app/models/database/bioproject/file_validator.rb deleted file mode 100644 index 0ebda794a..000000000 --- a/app/models/database/bioproject/file_validator.rb +++ /dev/null @@ -1,17 +0,0 @@ -class Database::BioProject::FileValidator - include DDBJValidator - - def translate_error(error) - message = error.fetch(:message) - annotations = error.fetch(:annotation, []).index_by { _1.fetch(:key) } - - case error.fetch(:id) - when 'BP_R0002' - xsd_message = annotations.fetch('XSD error message').fetch(:value) - - "#{message} #{xsd_message}" - else - message - end - end -end diff --git a/app/models/database/bioproject/submitter.rb b/app/models/database/bioproject/submitter.rb deleted file mode 100644 index bf804637d..000000000 --- a/app/models/database/bioproject/submitter.rb +++ /dev/null @@ -1,292 +0,0 @@ -class Database::BioProject::Submitter - class Error < StandardError; end - class SubmissionIDOverflow < Error; end - - PROJECT_DATA_TYPES = { - 'Genome Sequencing' => 'genome_sequencing', - 'Clone Ends' => 'clone_ends', - 'Epigenomics' => 'epigenomics', - 'Exome' => 'exome', - 'Map' => 'map', - 'Metagenome' => 'metagenome', - 'Phenotype and Genotype' => 'phenotype_and_genotype', - 'Proteome' => 'proteome', - 'Random Survey' => 'random_survey', - 'Targeted Locus (Loci)' => 'targeted_locus_loci', - 'Transcriptome or Gene Expression' => 'transcriptome_or_gene_expression', - 'Variation' => 'variation', - 'Other' => 'other' - } - - def submit(submission) - user = submission.validation.user - submitter_id = user.uid - - BioProject::Record.transaction isolation: Rails.env.test? ? nil : :serializable do |tx| - begin - submission_id = next_submission_id - rescue SubmissionIDOverflow => e - tx.after_rollback do - BioProject::ActionHistory.create!( - action: "[repository:CreateNewSubmission] #{e.message}", - action_date: Time.current, - result: false, - action_level: 'fatal', - submitter_id: - ) - end - - raise - end - - tx.after_rollback do - BioProject::ActionHistory.create!( - action: '[repository:CreateNewSubmission] rollback transaction', - action_date: Time.current, - result: false, - action_level: 'error', - submitter_id: - ) - end - - bp_submission = BioProject::Submission.create!( - submission_id:, - submitter_id:, - status_id: :data_submitted, - form_status_flags: '000000' - ) - - content = submission.validation.objs.find_by!(_id: 'BioProject').file.download - doc = Nokogiri::XML.parse(content) - is_public = submission.visibility_public? - hold = doc.at('/PackageSet/Package/Submission/Submission/Description/Hold') - - if is_public && hold - raise Error, 'Visibility is public, but Hold exist in XML.' - elsif !is_public && !hold - raise Error, 'Visibility is private, but Hold does not exist in XML.' - end - - set_accession_and_archive doc, submission_id - set_release_date doc if is_public - set_tax_id doc - - bp_submission.create_project!( - project_type: 'primary', - status_id: is_public ? :public : :private, - release_date: is_public ? Time.current : nil, - dist_date: is_public ? Time.current : nil, - modified_date: Time.current - ) - - bp_submission.submission_data.insert_all submission_data_attrs(submission, doc) - - version = (BioProject::XML.where(submission_id:).order(version: :desc).pick(:version) || 0) + 1 - - bp_submission.xmls.create!( - content: doc.to_s, - version:, - registered_date: Time.current - ) - - bp_submission.action_histories.create!( - action: '[repository:CreateNewSubmission] Create new submission', - action_date: Time.current, - result: true, - action_level: 'info', - submitter_id: - ) - - tx.after_commit do - DRMDB::ExtEntity.create!( - acc_type: :study, - ref_name: submission_id, - status: :valid - ) do |entity| - entity.ext_permits.build( - submitter_id: - ) - end - end - end - end - - private - - def next_submission_id - submission_id = BioProject::Submission.order(submission_id: :desc).pick(:submission_id) || 'PSUB000000' - num = submission_id.delete_prefix('PSUB').to_i - - raise SubmissionIDOverflow, 'Number of submission surpass the upper limit' if num >= 999_999 - - "PSUB#{num.succ.to_s.rjust(6, '0')}" - end - - def set_accession_and_archive(doc, project_id) - archive_id = doc.at('/PackageSet/Package/Project/Project/ProjectID/ArchiveID') - - archive_id[:accession] = project_id - archive_id[:archive] = 'DDBJ' - end - - def set_release_date(doc) - doc.at('/PackageSet/Package/Submission/Submission/Description/Hold')&.remove - - if release_date = doc.at('/PackageSet/Package/Project/Project/ProjectDescr/ProjectReleaseDate') - if release_date.text.empty? - release_date.content = Time.current.iso8601 - end - else - # TODO: Error handling - end - end - - def set_tax_id(doc) - return if doc.at('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/Target/Organism/@taxID') - - organism_name = doc.at('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/Target/Organism/OrganismName').text - names = DRASearch::TaxName.where(search_name: organism_name, name_class: 'scientific name').order(:tax_id) - - tax_id = case names.size - when 0 - nil - when 1 - names.first.tax_id - else - ids = names.map { "[#{_1.tax_id}] #{_1.uniq_name}" }.join(', ') - - raise Error, "Organism name is ambiguous, please set one of the following taxonomy IDs: #{ids}" - end - - raise Error, 'No entry found for the given organism name.' unless tax_id - - doc.at('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/Target/Organism')[:taxID] = tax_id - end - - def submission_data_attrs(submission, doc) - [ - *doc.xpath('/PackageSet/Package/Submission/Submission/Description/Organization/Contact').flat_map.with_index(1) {|contact, i| - first_name = contact.at('Name/First') - last_name = contact.at('Name/Last') - email = contact[:email] - - [ - ['submitter', 'first_name', first_name&.text, i], - ['submitter', 'last_name', last_name&.text, i], - ['submitter', 'email', email, i] - ] - }, - - doc.at('/PackageSet/Package/Submission/Submission/Description/Organization/Name').then { - ['submitter', 'organization_name', _1&.text, -1] - }, - - doc.at('/PackageSet/Package/Submission/Submission/Description/Organization/@url').then { - ['submitter', 'organization_url', _1&.text, -1] - }, - - ['submitter', 'data_release', submission.visibility_public? ? 'nonhup' : 'hup', -1], - - doc.at('/PackageSet/Package/Project/Project/ProjectDescr/Title').then { - ['general_info', 'project_title', _1&.text, -1] - }, - - doc.at('/PackageSet/Package/Project/Project/ProjectDescr/Description').then { - ['general_info', 'public_description', _1&.text, -1] - }, - - *doc.xpath('/PackageSet/Package/Project/Project/ProjectDescr/ExternalLink').flat_map.with_index(1) {|link, i| - url = link.at('URL') - - [ - ['general_info', 'link_description', link[:label], i], - ['general_info', 'link_url', url&.text, i] - ] - }, - - *doc.xpath('/PackageSet/Package/Project/Project/ProjectDescr/Grant').flat_map.with_index(1) {|grant, i| - agency = grant.at('Agency') - abbr = grant.at('Agency/@abbr') - title = grant.at('Title') - - [ - ['general_info', 'agency', agency&.text, i], - ['general_info', 'agency_abbreviation', abbr&.text, i], - ['general_info', 'grant_id', grant[:GrantId], i], - ['general_info', 'grant_title', title&.text, i] - ] - }, - - *doc.xpath('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/ProjectDataTypeSet/DataType').flat_map.with_index(1) {|data_type, i| - if value = PROJECT_DATA_TYPES[data_type.text] - [ - ['project_type', 'project_data_type', value, i] - ] - else - [ - ['project_type', 'project_data_type', 'other', i], - ['project_type', 'project_data_type_description', data_type.text, i] - ] - end - }, - - *doc.at('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/Target').then { - [ - ['project_type', 'sample_scope', _1&.[](:sample_scope), -1], - ['project_type', 'material', _1&.[](:material), -1], - ['project_type', 'capture', _1&.[](:capture), -1] - ] - }, - - *doc.at('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/Method').then { - method_type = _1&.[](:method_type) - description = method_type == 'eOther' ? _1&.text : nil - - [ - ['project_type', 'methodology', method_type, -1], - ['project_type', 'methodology_description', description, -1] - ] - }, - - *doc.xpath('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/Objectives').map.with_index(1) {|objectives, i| - data = objectives.at('Data') - - ['project_type', 'objective', data&.[](:data_type), i] - }, - - doc.at('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/Target/Organism/OrganismName').then { - ['target', 'organism_name', _1&.text, -1] - }, - - doc.at('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/Target/Organism/@taxID').then { - ['target', 'taxonomy_id', _1.text == '0' ? nil : _1.text, -1] - }, - - doc.at('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/Target/Organism/Strain').then { - ['target', 'strain_breed_cultivar', _1&.text, -1] - }, - - doc.at('/PackageSet/Package/Project/Project/ProjectType/ProjectTypeSubmission/Target/Organism/Label').then { - ['target', 'isolate_name_or_label', _1&.text, -1] - }, - - *doc.xpath('/PackageSet/Package/Project/Project/ProjectDescr/Publication').map.with_index(1) {|publication, i| - case dbtype = publication.at('Reference/DbType')&.text - when 'ePubmed' - ['publication', 'pubmed_id', publication[:id], i] - when 'eDOI' - ['publication', 'doi', publication[:id], i] - else - raise Error, "Unsupported publication type: #{dbtype.inspect}" - end - } - ].compact.map {|form_name, data_name, data_value, t_order| - { - form_name:, - data_name:, - data_value:, - t_order: - } - } - end -end diff --git a/app/models/database/biosample.rb b/app/models/database/biosample.rb deleted file mode 100644 index 15f26374c..000000000 --- a/app/models/database/biosample.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Database::BioSample - def self.build_param(params) - nil - end -end diff --git a/app/models/database/biosample/file_validator.rb b/app/models/database/biosample/file_validator.rb deleted file mode 100644 index a90406110..000000000 --- a/app/models/database/biosample/file_validator.rb +++ /dev/null @@ -1,40 +0,0 @@ -class Database::BioSample::FileValidator - include DDBJValidator - - def translate_error(error) - message = error.fetch(:message) - annotations = error.fetch(:annotation, []).index_by { _1.fetch(:key) } - sample_name = annotations.dig('Sample name', :value) - - case error.fetch(:id) - when 'BS_R0003' - %(The Sample title is not unique for the Sample name "#{sample_name}". Please provide a unique Sample title.) - when 'BS_R0013' - key = annotations.fetch('Attribute').fetch(:value) - value = annotations.fetch('Attribute value').fetch(:value) - suggested = annotations.fetch('Suggested value').fetch(:suggested_value).first - - %(Invalid data format. The "#{key}" attribute value is not valid for the Sample name "#{sample_name}". Please replace "#{value}" to "#{suggested}".) - when 'BS_R0015' - host = annotations.fetch('host').fetch(:value) - - %(Invalid host organism name. The "host" attribute value is not valid for the Sample name "#{sample_name}". Please correct "#{host}".) - when 'BS_R0045' - organism = annotations.fetch('organism').fetch(:value) - - %(Warning about "organism" for the Sample name "#{sample_name}". Please correct "#{organism}". If applicable, the taxonomy id will be automatically filled and the organism will be corrected to the scientific name. When the organism(s) is novel, please enter proposed name(s) in the organism, leave the taxonomy id empty and submit the BioSample.) - when 'BS_R0098' - xsd_message = annotations.fetch('message').fetch(:value) - - "#{message} #{xsd_message}" - when 'BS_R0100' - key = annotations.fetch('Attribute name').fetch(:value) - value = annotations.fetch('Attribute value').fetch(:value) - suggested = annotations.fetch('Suggested value').fetch(:suggested_value).first - - %(Missing values are not neccesary for optional attributes. Leave values empty when there is no information. The "#{key}" attribute value is not valid for the Sample name "#{sample_name}". Please replace "#{value}" to "#{suggested}".) - else - message - end - end -end diff --git a/app/models/database/biosample/submitter.rb b/app/models/database/biosample/submitter.rb deleted file mode 100644 index 616724fe3..000000000 --- a/app/models/database/biosample/submitter.rb +++ /dev/null @@ -1,291 +0,0 @@ -using FetchRaiseError - -class Database::BioSample::Submitter - class SubmissionIDOverflow < StandardError; end - - def submit(submission) - user = submission.validation.user - submitter_id = user.uid - - BioSample::Record.transaction isolation: Rails.env.test? ? nil : :serializable do |tx| - begin - submission_id = next_submission_id - rescue SubmissionIDOverflow - tx.after_rollback do - BioSample::OperationHistory.create!( - type: :fatal, - summary: '[repository:CreateNewSubmission] Number of submission surpass the upper limit', - date: Time.current, - submitter_id: - ) - end - - raise - end - - tx.after_rollback do - BioSample::OperationHistory.create!( - type: :error, - summary: '[repository:CreateNewSubmission] rollback transaction', - date: Time.current, - submitter_id: - ) - end - - content = submission.validation.objs.find_by!(_id: 'BioSample').file.download - doc = Nokogiri::XML.parse(content) - - organization = organization(doc) - organization_url = organization_url(doc) - attributes_assoc = attributes_assoc(doc) - package_id = package_id(doc) - - package_attributes(package_id) => { package_group:, env_package: } - - bs_submission_form = BioSample::SubmissionForm.create!( - submission_id:, - submitter_id:, - status_id: :data_submitted, - organization:, - organization_url:, - release_type: submission.visibility_public? ? 'release' : 'hold', - attribute_file_name: "#{submission_id}.tsv", - attribute_file: attribute_file(attributes_assoc), - comment: comment(doc), - package_group:, - package: package_id, - env_package: - ) - - bs_submission = bs_submission_form.create_submission!( - submitter_id:, - organization:, - organization_url:, - charge_id: 1 # NO_CHARGE - ) - - contacts = contacts(doc) - - bs_submission_form.contact_forms.insert_all contacts.map.with_index(1) {|contact, i| - { - **contact, - seq_no: i - } - } - - bs_submission.contacts.insert_all! contacts.map.with_index(1) {|contact, i| - { - **contact, - seq_no: i - } - } - - links = links(doc) - - bs_submission_form.link_forms.insert_all! links.map.with_index(1) {|link, i| - { - **link, - seq_no: i - } - } - - sample_names(doc).each do |sample_name| - sample = bs_submission.samples.create!( - sample_name:, - release_type: submission.visibility_public? ? 'release' : 'hold', - release_date: nil, - package_group:, - package: package_id, - env_package:, - status_id: :submission_accepted - ) - - attributes = attributes_assoc.fetch(sample_name) - - sample._attributes.insert_all! attributes.map.with_index(1) {|(name, value), i| - { - attribute_name: name, - attribute_value: value, - seq_no: i - } - } - - sample.links.insert_all! links.map.with_index(1) {|link, i| - { - **link, - seq_no: i - } - } - - version = (sample.xmls.maximum(:version) || 0) + 1 - xml = doc.at_xpath("/BioSampleSet/BioSample[Description/SampleName='#{sample_name}']") - - sample.xmls.create!( - version:, - content: xml.to_xml - ) - - tx.after_commit do - DRMDB::ExtEntity.create!( - acc_type: :sample, - ref_name: sample.smp_id, - status: :valid - ) do |entity| - entity.ext_permits.build( - submitter_id: - ) - end - end - end - - BioSample::OperationHistory.create!( - type: :info, - summary: '[repository:CreateNewSubmission] Create new submission', - date: Time.current, - submitter_id:, - submission_id: - ) - end - end - - private - - def next_submission_id - submission_id = BioSample::SubmissionForm.order(submission_id: :desc).pick(:submission_id) || 'SSUB000000' - num = submission_id.delete_prefix('SSUB').to_i - - raise SubmissionIDOverflow if num >= 999_999 - - "SSUB#{num.succ.to_s.rjust(6, '0')}" - end - - def organization(doc) - doc.xpath('/BioSampleSet/BioSample').map {|biosample| - biosample.at('Owner/Name')&.text - }.then {|orgnaizations| - raise "Inconsistent Owner/Name: #{orgnaizations.inspect}" if orgnaizations.uniq.size > 1 - - orgnaizations.first - } - end - - def organization_url(doc) - doc.xpath('/BioSampleSet/BioSample').map {|biosample| - biosample.at('Owner/Name/@url')&.text - }.then {|organization_urls| - raise "Inconsistent Owner/Name/@url: #{organization_urls.inspect}" if organization_urls.uniq.size > 1 - - organization_urls.first - } - end - - def package_id(doc) - doc.xpath('/BioSampleSet/BioSample').map {|biosample| - biosample.at('Models/Model')&.text - }.then {|models| - raise "Inconsistent Models/Model: #{models.inspect}" if models.uniq.size > 1 - - models.first - } - end - - def package_attributes(package_id) - res = Retriable.with_context(:fetch) { - Fetch::API.fetch("#{Rails.application.config_for(:app).validator_url!}/package_and_group_list").ensure_ok - } - - body = res.json - - collect_packages = ->(packages) { - packages + packages.flat_map { - collect_packages.call(_1.fetch(:package_list, [])) - } - } - - packages = collect_packages.call(body) - - packages_assoc = packages.select { _1.fetch(:type) == 'package' }.index_by { _1.fetch(:package_id) } - package_groups_assoc = packages.select { _1.fetch(:type) == 'package_group' }.index_by { _1.fetch(:package_group_uri) } - - package = packages_assoc.fetch(package_id) - package_group = package_groups_assoc.fetch(package.fetch(:parent_package_group_uri)) - - { - package_group: package_group.fetch(:package_group_id), - env_package: package.fetch(:env_package) - } - end - - def attributes_assoc(doc) - doc.xpath('/BioSampleSet/BioSample').map {|biosample| - biosample.xpath('Attributes/Attribute').map {|attribute| - [ - attribute[:attribute_name], - attribute.text - ] - }.to_h - }.then {|attributes_list| - keys_list = attributes_list.map(&:keys) - - raise "Inconsistent Attributes/Attribute/@attribute_name: #{keys_list.inspect}" if keys_list.uniq.size > 1 - - attributes_list.index_by { _1.fetch('sample_name') } - } - end - - def attribute_file(assoc) - CSV.generate(col_sep: "\t") {|tsv| - assoc.values.each_with_index do |attributes, i| - tsv << attributes.keys if i.zero? - tsv << attributes.values - end - } - end - - def comment(doc) - doc.xpath('/BioSampleSet/BioSample').map {|biosample| - biosample.at('Description/Comment/Paragraph')&.text - }.then {|comments| - raise "Inconsistent Description/Comment/Paragraph: #{comments.inspect}" if comments.uniq.size > 1 - - comments.first - } - end - - def contacts(doc) - doc.xpath('/BioSampleSet/BioSample').map {|biosample| - biosample.xpath('Owner/Contacts/Contact').map {|contact| - { - first_name: contact.at('Name/First')&.text, - last_name: contact.at('Name/Last')&.text, - email: contact[:email] - } - } - }.then {|contacts_list| - raise "Inconsistent Owner/Contacts/Contact: #{contacts_list.inspect}" if contacts_list.uniq.size > 1 - - contacts_list.first - } - end - - def links(doc) - doc.xpath('/BioSampleSet/BioSample').map {|biosample| - biosample.xpath('Links/Link').map {|link| - { - description: link[:label], - url: link.text - } - } - }.then {|links_list| - raise "Inconsistent Links/Link: #{links_list.inspect}" if links_list.uniq.size > 1 - - links_list.first - } - end - - def sample_names(doc) - doc.xpath('/BioSampleSet/BioSample').map {|biosample| - biosample.xpath('Description/SampleName')&.text - } - end -end diff --git a/app/models/database/dra.rb b/app/models/database/dra.rb deleted file mode 100644 index 50ae3b7fb..000000000 --- a/app/models/database/dra.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Database::DRA - def self.build_param(params) - nil - end -end diff --git a/app/models/database/dra/file_validator.rb b/app/models/database/dra/file_validator.rb deleted file mode 100644 index ef3674a53..000000000 --- a/app/models/database/dra/file_validator.rb +++ /dev/null @@ -1,48 +0,0 @@ -class Database::DRA::FileValidator - def validate(validation) - objs = validation.objs.without_base.index_by(&:_id) - - Dir.mktmpdir do |tmpdir| - tmpdir = Pathname.new(tmpdir) - - objs.values.each do |obj| - next unless obj - - obj.file.open do |file| - FileUtils.mv file.path, tmpdir.join("example-0001_dra_#{obj._id}.xml") - end - end - - Dir.chdir tmpdir do - env = { - 'BUNDLE_GEMFILE' => Rails.root.join('Gemfile').to_s - } - - out, status = Open3.capture2e(env, *%w[bundle exec validate_meta_dra -a example -i 0001 --machine-readable]) - - raise out unless status.success? - - errors = JSON.parse(out, symbolize_names: true).group_by { _1.fetch(:object_id) } - - validation.objs.without_base.group_by(&:_id).each do |obj_id, objs| - if errs = errors[obj_id] - objs.each do |obj| - obj.validity_invalid! - - errs.each do |err| - obj.validation_details.create!( - severity: 'error', - message: err.fetch(:message) - ) - end - end - else - objs.each do |obj| - obj.validity_valid! - end - end - end - end - end - end -end diff --git a/app/models/database/dra/submitter.rb b/app/models/database/dra/submitter.rb deleted file mode 100644 index 648446b73..000000000 --- a/app/models/database/dra/submitter.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Database::DRA::Submitter - def submit(submission) - # do nothing - end -end diff --git a/app/models/database/gea.rb b/app/models/database/gea.rb deleted file mode 100644 index a3892685c..000000000 --- a/app/models/database/gea.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Database::GEA - def self.build_param(params) - nil - end -end diff --git a/app/models/database/gea/file_validator.rb b/app/models/database/gea/file_validator.rb deleted file mode 100644 index 3799e401b..000000000 --- a/app/models/database/gea/file_validator.rb +++ /dev/null @@ -1,44 +0,0 @@ -class Database::GEA::FileValidator - def validate(validation) - objs = validation.objs.without_base.index_by(&:_id) - - validation.write_files_to_tmp do |tmpdir| - Dir.chdir tmpdir do - idf, sdrf = objs.fetch_values('IDF', 'SDRF').map(&:path) - - cmd = %W[bundle exec mb-validate --machine-readable -i #{idf} -s #{sdrf}].then { - if objs.key?('RawDataFile') || objs.key?('ProcessedDataFile') - _1 + %w[-d] - else - _1 - end - } - - out, status = Open3.capture2e({ - 'BUNDLE_GEMFILE' => Rails.root.join('Gemfile').to_s - }, *cmd) - - raise out unless status.success? - - errors = JSON.parse(out, symbolize_names: true).group_by { _1.fetch(:object_id) } - - validation.objs.without_base.group_by(&:_id).each do |obj_id, objs| - if errs = errors[obj_id] - objs.each do |obj| - # since this validator is a provisional implementation, validity should always be 'valid' - obj.validity_valid! - - errs.each do |err| - obj.validation_details.create! err.slice(:code, :severity, :message) - end - end - else - objs.each do |obj| - obj.validity_valid! - end - end - end - end - end - end -end diff --git a/app/models/database/gea/submitter.rb b/app/models/database/gea/submitter.rb deleted file mode 100644 index 38ee83777..000000000 --- a/app/models/database/gea/submitter.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Database::GEA::Submitter - def submit(submission) - # do nothing - end -end diff --git a/app/models/database/jvar.rb b/app/models/database/jvar.rb deleted file mode 100644 index e2f1f1045..000000000 --- a/app/models/database/jvar.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Database::JVar - def self.build_param(params) - nil - end -end diff --git a/app/models/database/jvar/file_validator.rb b/app/models/database/jvar/file_validator.rb deleted file mode 100644 index 74c02d653..000000000 --- a/app/models/database/jvar/file_validator.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Database::JVar::FileValidator - def validate(validation) - validation.objs.without_base.each(&:validity_valid!) - end -end diff --git a/app/models/database/jvar/submitter.rb b/app/models/database/jvar/submitter.rb deleted file mode 100644 index 7d82c72e3..000000000 --- a/app/models/database/jvar/submitter.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Database::JVar::Submitter - def submit(submission) - # do nothing - end -end diff --git a/app/models/database/metabobank.rb b/app/models/database/metabobank.rb deleted file mode 100644 index 070303017..000000000 --- a/app/models/database/metabobank.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Database::MetaboBank - def self.build_param(params) - nil - end -end diff --git a/app/models/database/metabobank/file_validator.rb b/app/models/database/metabobank/file_validator.rb deleted file mode 100644 index a05b59e34..000000000 --- a/app/models/database/metabobank/file_validator.rb +++ /dev/null @@ -1,55 +0,0 @@ -class Database::MetaboBank::FileValidator - def validate(validation) - objs = validation.objs.without_base.index_by(&:_id) - - validation.write_files_to_tmp do |tmpdir| - Dir.chdir tmpdir do - idf, sdrf = objs.fetch_values('IDF', 'SDRF').map(&:path) - - cmd = %W[bundle exec mb-validate --machine-readable -i #{idf} -s #{sdrf}].then { - if bs = objs['BioSample'] - _1 + %W[-t #{bs.path}] - else - _1 - end - }.then { - if objs.key?('MAF') || objs.key?('RawDataFile') || objs.key?('ProcessedDataFile') - _1 + %w[-d] - else - _1 - end - } - - out, status = Open3.capture2e({ - 'BUNDLE_GEMFILE' => Rails.root.join('Gemfile').to_s - }, *cmd) - - raise out unless status.success? - - errors = JSON.parse(out, symbolize_names: true).group_by { _1.fetch(:object_id) } - - validation.objs.without_base.group_by(&:_id).each do |obj_id, objs| - if errs = errors[obj_id] - validity = if errs.any? { _1[:severity] == 'error' } - 'invalid' - else - 'valid' - end - - objs.each do |obj| - obj.update! validity: validity - - errs.each do |err| - obj.validation_details.create! err.slice(:code, :severity, :message) - end - end - else - objs.each do |obj| - obj.validity_valid! - end - end - end - end - end - end -end diff --git a/app/models/database/metabobank/submitter.rb b/app/models/database/metabobank/submitter.rb deleted file mode 100644 index 517f92f1c..000000000 --- a/app/models/database/metabobank/submitter.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Database::MetaboBank::Submitter - def submit(submission) - # do nothing - end -end diff --git a/app/models/database/trad.rb b/app/models/database/trad.rb deleted file mode 100644 index fb5b015df..000000000 --- a/app/models/database/trad.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Database::Trad - def self.build_param(params) - nil - end -end diff --git a/app/models/database/trad/file_validator.rb b/app/models/database/trad/file_validator.rb deleted file mode 100644 index 2ed17e059..000000000 --- a/app/models/database/trad/file_validator.rb +++ /dev/null @@ -1,108 +0,0 @@ -class Database::Trad::FileValidator - class MissingContactPersonInformation < StandardError; end - class DuplicateContactPersonInformation < StandardError; end - - include TradValidation - - ASSOC = { - 'Sequence' => %w[.fasta .seq.fa .fa .fna .seq], - 'Annotation' => %w[.ann .annt.tsv .ann.txt] - } - - def validate(validation) - objs = validation.objs.without_base - - validate_ext objs, ASSOC - validate_nwise objs, ASSOC - validate_seq objs - validate_ann objs - - objs.each do |obj| - if obj.validation_details.empty? - obj.validity_valid! - else - obj.validity_invalid! - end - end - end - - private - - def validate_ann(objs) - anns = objs.select { _1._id == 'Annotation' } - - return if anns.empty? - - assoc = anns.map {|obj| - begin - contact_person = extract_contact_person_in_ann(obj.file) - rescue MissingContactPersonInformation - obj.validation_details.create!( - severity: 'error', - message: 'Contact person information (contact, email, institute) is missing.' - ) - rescue DuplicateContactPersonInformation - obj.validation_details.create!( - severity: 'error', - message: 'Contact person information (contact, email, institute) is duplicated.' - ) - end - - - [obj, contact_person] - } - - _, first_contact_person = assoc.first - - assoc.each do |obj, contact_person| - unless first_contact_person == contact_person - obj.validation_details.create!( - severity: 'error', - message: 'Contact person must be the same for all annotation files.' - ) - end - end - end - - def extract_contact_person_in_ann(file) - in_common = false - contact = nil - email = nil - institute = nil - - file.download.each_line chomp: true do |line| - entry, _feature, _location, qualifier, value = line.split("\t") - - break if in_common && entry.present? - - in_common = entry == 'COMMON' if entry.present? - - next unless in_common - - case qualifier - when 'contact' - raise DuplicateContactPersonInformation if contact - - contact = value - when 'email' - raise DuplicateContactPersonInformation if email - - email = value - when 'institute' - raise DuplicateContactPersonInformation if institute - - institute = value - else - # do nothing - end - end - - raise MissingContactPersonInformation unless contact && email && institute - - { - contact:, - email:, - institute: - } - end -end diff --git a/app/models/database/trad2.rb b/app/models/database/trad2.rb deleted file mode 100644 index 12d2da72a..000000000 --- a/app/models/database/trad2.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Database::Trad2 - def self.build_param(params) - nil - end -end diff --git a/app/models/database/trad2/file_validator.rb b/app/models/database/trad2/file_validator.rb deleted file mode 100644 index e0034f726..000000000 --- a/app/models/database/trad2/file_validator.rb +++ /dev/null @@ -1,37 +0,0 @@ -class Database::Trad2::FileValidator - include TradValidation - - ASSOC = { - 'Sequence' => %w[.fasta .seq.fa .fa .fna .seq], - 'Annotation' => %w[.gff], - 'Metadata' => %w[.tsv] - } - - def validate(validation) - objs = validation.objs.without_base - - validate_ext objs, ASSOC - validate_nwise objs, ASSOC - validate_seq objs - validate_ann objs - - objs.each do |obj| - if obj.validation_details.empty? - obj.validity_valid! - else - obj.validity_invalid! - end - end - end - - def validate_ann(objs) - objs.select { _1._id == 'Annotation' }.each do |obj| - NoodlesGFF.parse obj.file.download - rescue NoodlesGFF::Error => e - obj.validation_details.create!( - severity: 'error', - message: e.message - ) - end - end -end diff --git a/app/models/database/trad2/submitter.rb b/app/models/database/trad2/submitter.rb deleted file mode 100644 index cf0a92921..000000000 --- a/app/models/database/trad2/submitter.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Database::Trad2::Submitter - def submit(submission) - # do nothing - end -end diff --git a/app/models/database/trad/ddbj_record_validator.rb b/app/models/ddbj_record_validator.rb similarity index 65% rename from app/models/database/trad/ddbj_record_validator.rb rename to app/models/ddbj_record_validator.rb index 9ceecd0ad..c8d087b7b 100644 --- a/app/models/database/trad/ddbj_record_validator.rb +++ b/app/models/ddbj_record_validator.rb @@ -1,12 +1,45 @@ -class Database::Trad::DDBJRecordValidator - def validate(validation) +module DDBJRecordValidator + module_function + + def validate(subject) + ActiveRecord::Base.transaction do + subject.validating! + subject.create_validation! + end + + ActiveRecord::Base.transaction do + begin + _validate subject + rescue => e + Rails.error.report e + + subject.validation_failed! + + subject.validation.details.create!( + severity: :error, + message: e.message + ) + else + if subject.validation.details.where(severity: :error).exists? + subject.validation_failed! + else + subject.ready_to_apply! + end + end + ensure + subject.validation.update! progress: :finished, finished_at: Time.current + end + end + + def _validate(subject) details = [] - obj = validation.objs.without_base.last - record = JSON.parse(obj.file.download, symbolize_names: true) + filename = subject.validation.subject.ddbj_record.filename.to_s + record = JSON.parse(subject.validation.subject.ddbj_record.download, symbolize_names: true) app_number = record.dig(:submission, :application_identification, :application_number_text) unless app_number&.match?(%r(\A\d{4}[-/]\d{6}\z)) details << { + filename:, entry_id: nil, code: 'SB-02001', severity: 'error', @@ -17,12 +50,17 @@ def validate(validation) Array(record.dig(:sequence, :entries)).each do |entry| entry_id = entry[:id] - details.concat validate_qualifiers(entry[:source_qualifiers], obj:, entry_id:, feature: :source) + details.concat validate_qualifiers(entry[:source_qualifiers], **{ + filename:, + entry_id:, + feature: :source + }) seq = entry[:sequence].to_s if seq.empty? details << { + filename:, entry_id:, code: 'SB-02006', severity: 'error', @@ -32,6 +70,7 @@ def validate(validation) if seq.match?(/\AN+\z/i) details << { + filename:, entry_id:, code: 'SB-02007', severity: 'error', @@ -41,6 +80,7 @@ def validate(validation) if seq.match?(/\AX+\z/i) details << { + filename:, entry_id:, code: 'SB-02008', severity: 'error', @@ -52,6 +92,7 @@ def validate(validation) if !aa && seq.match?(/[^acgtmrwsykvhdbn]/i) details << { + filename:, entry_id:, code: 'SB-02009', severity: 'error', @@ -66,6 +107,7 @@ def validate(validation) unless FeatureChecker.defined_feature?(fkey) details << { + filename:, entry_id:, code: 'SB-02003', severity: 'warning', @@ -73,28 +115,25 @@ def validate(validation) } end - details.concat validate_qualifiers(feature[:qualifiers], obj:, entry_id:, feature: fkey) + details.concat validate_qualifiers(feature[:qualifiers], **{ + filename:, + entry_id:, + feature: fkey + }) end rescue JSON::ParserError => e details << { + filename:, entry_id: nil, code: nil, severity: 'error', message: e.message } ensure - if details.any? { it[:severity] == 'error' } - obj.validity_invalid! - else - obj.validity_valid! - end - - obj.validation_details.insert_all! details + subject.validation.details.insert_all! details end - private - - def validate_qualifiers(quals, obj:, entry_id:, feature:) + def validate_qualifiers(quals, filename:, entry_id:, feature:) details = [] Array(quals).each do |qkey, entries| @@ -102,6 +141,7 @@ def validate_qualifiers(quals, obj:, entry_id:, feature:) unless FeatureChecker.defined_qualifier?(qkey) details << { + filename:, entry_id:, code: 'SB-02004', severity: 'warning', @@ -112,6 +152,7 @@ def validate_qualifiers(quals, obj:, entry_id:, feature:) entries.pluck(:value).each do |value| unless FeatureChecker.qualifier_value_presence_valid?(qkey, value) details << { + filename:, entry_id:, code: 'SB-02005', severity: 'error', diff --git a/app/models/obj.rb b/app/models/obj.rb deleted file mode 100644 index c21a531f9..000000000 --- a/app/models/obj.rb +++ /dev/null @@ -1,59 +0,0 @@ -using PathnameContain - -class Obj < ApplicationRecord - belongs_to :validation - - has_many :validation_details, -> { order(:id) }, dependent: :destroy - - has_one_attached :file - - scope :without_base, -> { where.not(_id: '_base') } - - enum :_id, DB.flat_map { it[:objects].values.flatten }.pluck(:id).uniq.concat(['_base']).index_by(&:to_sym) - enum :validity, %w[valid invalid error].index_by(&:to_sym), prefix: true - - validates :file, attached: true, unless: :base? - - validate :destination_must_not_be_malformed - validate :path_must_be_unique_in_request - - def base? = _id == '_base' - - def path - return nil if base? - - [destination, file.filename.sanitized].compact_blank.join('/') - end - - def validation_result - { - object_id: _id, - validity: validity, - details: validation_details.map { it.slice(:entry_id, :code, :severity, :message).symbolize_keys }, - - file: base? ? nil : { - path:, - url: Rails.application.routes.url_helpers.validation_file_url(validation, path) - } - } - end - - private - - def destination_must_not_be_malformed - return if base? - return unless destination - - tmp = Pathname.new('/tmp').join(SecureRandom.uuid) - - errors.add :destination, 'is malformed path' unless tmp.contain?(tmp.join(destination)) - end - - def path_must_be_unique_in_request - return if base? - - if validation.objs.to_a.without(self).any? { path == it.path } - errors.add :path, "is duplicated: #{path}" - end - end -end diff --git a/app/models/submission.rb b/app/models/submission.rb index 70a5c0383..9ef8a66cf 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -1,18 +1,10 @@ class Submission < ApplicationRecord - belongs_to :validation + has_one :request, dependent: :destroy, class_name: 'SubmissionRequest' + has_many :updates, dependent: :destroy, class_name: 'SubmissionUpdate' has_many :accessions, dependent: :destroy - delegated_type :param, types: %w[BioProjectSubmissionParam], optional: true, dependent: :destroy - - validates :validation_id, uniqueness: {message: 'is already submitted'} - - validate :validation_must_be_valid - validate :validation_finished_at_must_be_in_24_hours - - enum :progress, %w[waiting running finished canceled].index_by(&:to_sym) - enum :result, %w[success failure].index_by(&:to_sym) - enum :visibility, %w[public private].index_by(&:to_sym), prefix: true + has_one_attached :ddbj_record after_destroy do |submission| submission.dir.rmtree @@ -21,20 +13,6 @@ class Submission < ApplicationRecord def dir base = Rails.application.config_for(:app).repository_dir! - Pathname.new(base).join(validation.user.uid, 'submissions', id.to_s) - end - - private - - def validation_must_be_valid - unless validation.validity == 'valid' - errors.add :validation, 'must be valid' - end - end - - def validation_finished_at_must_be_in_24_hours - if validation.validity == 'valid' && validation.finished_at <= 1.day.ago - errors.add :validation, 'finished_at must be in 24 hours' - end + Pathname.new(base).join(request.user.uid, 'submissions', id.to_s) end end diff --git a/app/models/submission_request.rb b/app/models/submission_request.rb new file mode 100644 index 000000000..4b893f49f --- /dev/null +++ b/app/models/submission_request.rb @@ -0,0 +1,8 @@ +class SubmissionRequest < ApplicationRecord + include ValidationSubject + + belongs_to :user + belongs_to :submission, optional: true, inverse_of: :request + + has_one_attached :ddbj_record +end diff --git a/app/models/submission_update.rb b/app/models/submission_update.rb new file mode 100644 index 000000000..ac11cf65c --- /dev/null +++ b/app/models/submission_update.rb @@ -0,0 +1,7 @@ +class SubmissionUpdate < ApplicationRecord + include ValidationSubject + + belongs_to :submission, inverse_of: :updates + + has_one_attached :ddbj_record +end diff --git a/app/models/user.rb b/app/models/user.rb index 787f6912d..807fd6748 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -3,9 +3,10 @@ def self.generate_api_key SecureRandom.base58(32) end - has_many :validations, dependent: :destroy + has_many :submission_requests - has_many :submissions, through: :validations + has_many :submissions, through: :submission_requests + has_many :submission_updates, through: :submissions, source: :updates has_many :accessions, through: :submissions has_many :accession_renewals, through: :accessions, source: :renewals diff --git a/app/models/validation.rb b/app/models/validation.rb index 25d8299b7..e9bf76471 100644 --- a/app/models/validation.rb +++ b/app/models/validation.rb @@ -3,83 +3,32 @@ class Validation < ApplicationRecord class UnprocessableContent < StandardError; end - belongs_to :user + belongs_to :subject, polymorphic: true - has_one :submission, dependent: :destroy - - has_many :objs, -> { order(:id) }, dependent: :destroy do - def base - find { _1._id == '_base' } - end - end + has_many :details, dependent: :destroy, class_name: 'ValidationDetail' serialize :raw_result, coder: JSON - validates :db, inclusion: {in: DB.map { _1[:id] }} - - validates :started_at, presence: true, if: :running? validates :finished_at, presence: true, if: ->(validation) { validation.finished? || validation.canceled? } - enum :progress, %w[waiting running finished canceled].index_by(&:to_sym) - enum :via, %w[file ddbj_record].index_by(&:to_sym), prefix: true - - scope :validity, ->(*validities) { - return none if validities.empty? - - sql = validities.map {|validity| - case validity - when 'valid' - <<~SQL - NOT EXISTS ( - SELECT 1 FROM objs - WHERE objs.validation_id = validations.id - AND (objs.validity <> 'valid' OR objs.validity IS NULL) - ) - SQL - when 'invalid' - <<~SQL - NOT EXISTS ( - SELECT 1 FROM objs - WHERE objs.validation_id = validations.id - AND objs.validity = 'error' - ) AND objs.validity = 'invalid' - SQL - when 'error' - %(objs.validity = 'error') - when 'null' - <<~SQL - NOT EXISTS ( - SELECT 1 FROM objs - WHERE objs.validation_id = validations.id - AND (objs.validity <> 'valid' OR objs.validity <> 'invalid' OR objs.validity <> 'error') - ) - SQL - else - raise ArgumentError, validity - end - }.join(' OR ') - - joins(:objs).where(sql) - } + enum :progress, %w[running finished canceled].index_by(&:to_sym), validate: true scope :submitted, ->(submitted) { submitted ? where.associated(:submission) : where.missing(:submission) } - def validity - if objs.all?(&:validity_valid?) - 'valid' - elsif objs.any?(&:validity_error?) - 'error' - elsif objs.any?(&:validity_invalid?) - 'invalid' - else - nil - end - end + scope :with_validity, -> { + left_joins(:details).group(:id).select('validations.*', <<~SQL) + CASE + WHEN validations.progress != 'finished' THEN NULL + WHEN COUNT(CASE WHEN validation_details.severity = 'error' THEN 1 END) = 0 THEN 'valid' + ELSE 'invalid' + END AS validity + SQL + } def results - objs.map(&:validation_result) + subject.objs.map(&:validation_result) end def build_obj_from_path(relative_path, obj_schema:, destination:, user:) @@ -142,10 +91,10 @@ def write_files_to_tmp(&block) def write_submission_files(to:) to.tap(&:mkpath).join('validation-report.json').write JSON.pretty_generate(results) - objs.each do |obj| + subject.objs.each do |obj| obj_dir = to.join(obj._id) - if obj.base? + if obj._base? obj_dir.mkpath obj_dir.join('validation-report.json').write JSON.pretty_generate(obj.validation_result) else diff --git a/app/models/validation_detail.rb b/app/models/validation_detail.rb index 4b9879bf6..f4cdc2f36 100644 --- a/app/models/validation_detail.rb +++ b/app/models/validation_detail.rb @@ -1,3 +1,5 @@ class ValidationDetail < ApplicationRecord - belongs_to :obj + belongs_to :validation, inverse_of: :details + + enum :severity, %w[warning error].index_by(&:to_sym), validate: true end diff --git a/app/views/application/_blob.json.jb b/app/views/application/_blob.json.jb new file mode 100644 index 000000000..00c06c2ab --- /dev/null +++ b/app/views/application/_blob.json.jb @@ -0,0 +1,4 @@ +{ + filename: blob.filename, + url: rails_blob_url(blob) +} diff --git a/app/views/submission_requests/_submission_request.json.jb b/app/views/submission_requests/_submission_request.json.jb new file mode 100644 index 000000000..400b56c0c --- /dev/null +++ b/app/views/submission_requests/_submission_request.json.jb @@ -0,0 +1,12 @@ +{ + **submission_request.slice( + :id, + :status, + :error_message, + :created_at + ), + + ddbj_record: render('blob', blob: submission_request.ddbj_record), + validation: submission_request.validation_with_validity&.then { render(it) }, + submission: submission_request.submission&.then { render(it) } +} diff --git a/app/views/submission_requests/index.json.jb b/app/views/submission_requests/index.json.jb new file mode 100644 index 000000000..ede6ebebc --- /dev/null +++ b/app/views/submission_requests/index.json.jb @@ -0,0 +1 @@ +render(partial: 'submission_request', collection: @requests) diff --git a/app/views/submission_requests/show.json.jb b/app/views/submission_requests/show.json.jb new file mode 100644 index 000000000..895297cdb --- /dev/null +++ b/app/views/submission_requests/show.json.jb @@ -0,0 +1 @@ +render(@request) diff --git a/app/views/submission_updates/show.json.jb b/app/views/submission_updates/show.json.jb new file mode 100644 index 000000000..e4f10a0f5 --- /dev/null +++ b/app/views/submission_updates/show.json.jb @@ -0,0 +1,12 @@ +{ + **@update.slice( + :id, + :status, + :error_message, + :created_at + ), + + ddbj_record: render('blob', blob: @update.ddbj_record), + validation: @update.validation_with_validity&.then { render(it) }, + submission: render(@update.submission) +} diff --git a/app/views/submissions/_submission.json.jb b/app/views/submissions/_submission.json.jb index ce78c7f61..1e7d3327d 100644 --- a/app/views/submissions/_submission.json.jb +++ b/app/views/submissions/_submission.json.jb @@ -1,21 +1,37 @@ -# locals: (submission:, validation: false) +# locals: (submission:) { **submission.slice( :id, - :created_at, - :started_at, - :finished_at, - :progress, - :result, - :error_message, - :visibility + :created_at ), - db: submission.validation.db, - url: submission_url(submission), - accessions: render(partial: 'accessions/accession', collection: submission.accessions), + ddbj_record: submission.ddbj_record.then {|record| + record.attached? ? { + filename: record.filename, + url: rails_blob_url(record) + } : nil + }, - **submission.param&.as_json, - **(validation ? {validation: render(submission.validation)} : nil) + accessions: submission.accessions.map {|accession| + accession.slice( + :number, + :entry_id, + :version, + :last_updated_at + ) + }, + + updates: submission.updates.map {|update| + { + **update.slice( + :id, + :status, + :created_at + ), + + ddbj_record: render('blob', blob: update.ddbj_record), + validation: update.validation_with_validity&.then { render(it) } + } + } } diff --git a/app/views/submissions/index.json.jb b/app/views/submissions/index.json.jb index 8ea3f1df3..44e80de14 100644 --- a/app/views/submissions/index.json.jb +++ b/app/views/submissions/index.json.jb @@ -1 +1 @@ -render(partial: 'submission', collection: @submissions, locals: {validation: true}) +render(partial: 'submission', collection: @submissions) diff --git a/app/views/submissions/show.json.jb b/app/views/submissions/show.json.jb index 225011d2f..5c91a6c02 100644 --- a/app/views/submissions/show.json.jb +++ b/app/views/submissions/show.json.jb @@ -1 +1 @@ -render(@submission, validation: true) +render(@submission) diff --git a/app/views/validations/_validation.json.jb b/app/views/validations/_validation.json.jb index b412e15b4..7332c02b7 100644 --- a/app/views/validations/_validation.json.jb +++ b/app/views/validations/_validation.json.jb @@ -1,24 +1,19 @@ -# locals: (validation:, submission: false) - { - id: validation.id, - url: validation_url(validation), - - user: { - uid: validation.user.uid - }, - - db: validation.db, - created_at: validation.created_at, - started_at: validation.started_at, - finished_at: validation.finished_at, - progress: validation.progress, - validity: validation.validity, - objects: render('validations/objs', validation:), - results: validation.results, - raw_result: validation.raw_result, + **validation.slice( + :id, + :progress, + :created_at, + :finished_at, + :validity + ), - **(submission ? { - submission: validation.submission.then { it ? render(it) : nil } - } : nil) + details: validation.details.map {|detail| + detail.slice( + :filename, + :entry_id, + :code, + :severity, + :message + ) + } } diff --git a/config/initializers/active_storage.rb b/config/initializers/active_storage.rb index 6baad7e5c..4ca7e53b1 100644 --- a/config/initializers/active_storage.rb +++ b/config/initializers/active_storage.rb @@ -1 +1,5 @@ -Rails.application.config.active_storage.draw_routes = false unless Rails.env.test? +Rails.application.config.to_prepare do + ActiveStorage::DirectUploadsController.class_eval do + skip_forgery_protection + end +end diff --git a/config/initializers/cors.rb b/config/initializers/cors.rb index 32d86f24e..8165ae3ba 100644 --- a/config/initializers/cors.rb +++ b/config/initializers/cors.rb @@ -7,5 +7,10 @@ expose: %w[Link Current-Page Page-Items Total-Pages Total-Count], methods: %i[get post put patch delete options head] } + + resource '/rails/active_storage/direct_uploads', **{ + headers: :any, + methods: %i[post] + } end end diff --git a/config/routes.rb b/config/routes.rb index adc3b6f19..d18624a00 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -11,18 +11,15 @@ resource :me, only: %i[show] - resources :validations, only: %i[index show destroy] do - scope module: 'validations' do - collection do - resource :via_file, only: %i[create] - resource :via_ddbj_record, only: %i[create] - end - - get 'files/*path' => 'files#show', format: false, as: 'file' - end + resources :submission_requests, only: %i[index show create] do + resource :submission, only: :create end - resources :submissions, only: %i[index show create] do + resources :submission_updates, only: %i[show] + + resources :submissions, only: %i[index show] do + resources :updates, only: %i[create], controller: 'submission_updates' + resources :accessions, only: %i[show update], param: :number, shallow: true do resources :accession_renewals, only: %i[show create] do scope module: 'accession_renewals' do diff --git a/db/migrate/20251119073632_create_submission_requests.rb b/db/migrate/20251119073632_create_submission_requests.rb new file mode 100644 index 000000000..d65ebb1d9 --- /dev/null +++ b/db/migrate/20251119073632_create_submission_requests.rb @@ -0,0 +1,58 @@ +class CreateSubmissionRequests < ActiveRecord::Migration[8.1] + def change + create_table :submission_requests do |t| + t.references :user, null: false, foreign_key: true + t.references :submission, null: true, foreign_key: true + + t.integer :status, null: false, default: 0 + t.string :error_message + + t.timestamps + end + + create_table :submission_updates do |t| + t.references :submission, null: false, foreign_key: true + + t.integer :status, null: false, default: 0 + t.string :error_message + + t.timestamps + end + + change_table :submissions do |t| + t.remove :error_message + t.remove :finished_at + t.remove :param_id + t.remove :param_type + t.remove :progress + t.remove :result + t.remove :started_at + t.remove :validation_id + t.remove :visibility + end + + change_table :validations do |t| + t.references :subject, null: false, polymorphic: true + + t.remove :user_id + t.remove :db + t.remove :via + t.remove :started_at + end + + change_column_default :validations, :progress, 'running' + + change_table :validation_details do |t| + t.references :validation, null: false, foreign_key: true + + t.string :filename + + t.remove :obj_id + end + + drop_table :accession_renewal_validation_details + drop_table :accession_renewals + drop_table :objs + drop_table :bioproject_submission_params + end +end diff --git a/db/schema.rb b/db/schema.rb index 70d38dcd3..53deb9d0d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,27 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2025_10_29_071815) do - create_table "accession_renewal_validation_details", force: :cascade do |t| - t.datetime "created_at", null: false - t.string "message", null: false - t.integer "renewal_id", null: false - t.string "severity", null: false - t.datetime "updated_at", null: false - t.index ["renewal_id"], name: "index_accession_renewal_validation_details_on_renewal_id" - end - - create_table "accession_renewals", force: :cascade do |t| - t.integer "accession_id", null: false - t.datetime "created_at", null: false - t.datetime "finished_at" - t.string "progress", default: "waiting", null: false - t.datetime "started_at" - t.datetime "updated_at", null: false - t.string "validity" - t.index ["accession_id"], name: "index_accession_renewals_on_accession_id" - end - +ActiveRecord::Schema[8.1].define(version: 2025_11_19_073632) do create_table "accessions", force: :cascade do |t| t.datetime "created_at", null: false t.string "entry_id", null: false @@ -72,46 +52,39 @@ t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end - create_table "bioproject_submission_params", force: :cascade do |t| + create_table "sequences", force: :cascade do |t| t.datetime "created_at", null: false - t.boolean "umbrella", null: false + t.integer "digits", null: false + t.bigint "next", default: 1, null: false + t.string "prefix", null: false + t.string "scope", null: false t.datetime "updated_at", null: false + t.index ["scope"], name: "index_sequences_on_scope", unique: true end - create_table "objs", force: :cascade do |t| - t.string "_id", null: false + create_table "submission_requests", force: :cascade do |t| t.datetime "created_at", null: false - t.string "destination" + t.string "error_message" + t.integer "status", default: 0, null: false + t.integer "submission_id" t.datetime "updated_at", null: false - t.bigint "validation_id", null: false - t.string "validity" - t.index ["validation_id"], name: "index_objs_on_validation_id" - t.index ["validity"], name: "index_objs_on_validity" + t.integer "user_id", null: false + t.index ["submission_id"], name: "index_submission_requests_on_submission_id" + t.index ["user_id"], name: "index_submission_requests_on_user_id" end - create_table "sequences", force: :cascade do |t| + create_table "submission_updates", force: :cascade do |t| t.datetime "created_at", null: false - t.integer "digits", null: false - t.bigint "next", default: 1, null: false - t.string "prefix", null: false - t.string "scope", null: false + t.string "error_message" + t.integer "status", default: 0, null: false + t.integer "submission_id", null: false t.datetime "updated_at", null: false - t.index ["scope"], name: "index_sequences_on_scope", unique: true + t.index ["submission_id"], name: "index_submission_updates_on_submission_id" end create_table "submissions", force: :cascade do |t| t.datetime "created_at", null: false - t.string "error_message" - t.datetime "finished_at" - t.string "param_id" - t.string "param_type" - t.string "progress", default: "waiting", null: false - t.string "result" - t.datetime "started_at" t.datetime "updated_at", null: false - t.bigint "validation_id", null: false - t.string "visibility", null: false - t.index ["validation_id"], name: "index_submissions_on_validation_id", unique: true end create_table "users", force: :cascade do |t| @@ -128,36 +101,32 @@ t.string "code" t.datetime "created_at", null: false t.string "entry_id" + t.string "filename" t.string "message" - t.bigint "obj_id", null: false t.string "severity" t.datetime "updated_at", null: false - t.index ["obj_id"], name: "index_validation_details_on_obj_id" + t.integer "validation_id", null: false + t.index ["validation_id"], name: "index_validation_details_on_validation_id" end create_table "validations", force: :cascade do |t| t.datetime "created_at", null: false - t.string "db", null: false t.datetime "finished_at" - t.string "progress", default: "waiting", null: false + t.string "progress", default: "running", null: false t.string "raw_result" - t.datetime "started_at" + t.integer "subject_id", null: false + t.string "subject_type", null: false t.datetime "updated_at", null: false - t.bigint "user_id", null: false - t.string "via", null: false t.index ["created_at"], name: "index_validations_on_created_at" - t.index ["db"], name: "index_validations_on_db" t.index ["progress"], name: "index_validations_on_progress" - t.index ["user_id"], name: "index_validations_on_user_id" + t.index ["subject_type", "subject_id"], name: "index_validations_on_subject" end - add_foreign_key "accession_renewal_validation_details", "accession_renewals", column: "renewal_id" - add_foreign_key "accession_renewals", "accessions" add_foreign_key "accessions", "submissions" add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" - add_foreign_key "objs", "validations" - add_foreign_key "submissions", "validations" - add_foreign_key "validation_details", "objs" - add_foreign_key "validations", "users" + add_foreign_key "submission_requests", "submissions" + add_foreign_key "submission_requests", "users" + add_foreign_key "submission_updates", "submissions" + add_foreign_key "validation_details", "validations" end diff --git a/spec/factories/submission_requests.rb b/spec/factories/submission_requests.rb new file mode 100644 index 000000000..9a100338d --- /dev/null +++ b/spec/factories/submission_requests.rb @@ -0,0 +1,9 @@ +FactoryBot.define do + factory :submission_request do + db { 'JVar' } + + after :create do |request| + create :obj, owner: request, _id: '_base', file: nil + end + end +end diff --git a/spec/factories/validations.rb b/spec/factories/validations.rb index 3a651be69..b6bf005c3 100644 --- a/spec/factories/validations.rb +++ b/spec/factories/validations.rb @@ -1,30 +1,12 @@ FactoryBot.define do factory :validation do - transient do - validity { nil } - end - - user - - db { DB.map { _1[:id] }.sample } - via { :file } - after :build do |validation| - if validation.running? || validation.finished? || validation.canceled? - validation.started_at = '2024-01-02' unless validation.started_at - end - if validation.finished? || validation.canceled? validation.finished_at = '2024-01-03' unless validation.finished_at end end - after :create do |validation, evaluator| - create :obj, validation:, _id: '_base', file: nil, validity: evaluator.validity - end - trait :valid do - validity { 'valid' } progress { 'finished' } finished_at { Time.current } end diff --git a/spec/requests/submission_requests_spec.rb b/spec/requests/submission_requests_spec.rb new file mode 100644 index 000000000..6ba5fad59 --- /dev/null +++ b/spec/requests/submission_requests_spec.rb @@ -0,0 +1,73 @@ +require 'rails_helper' + +RSpec.describe '/submission_requests', type: :request do + let(:user) { create(:user) } + let(:default_headers) { {'Authorization' => "Bearer #{user.api_key}"} } + + example 'index' do + get submission_requests_path + + expect(response).to have_http_status(:ok) + end + + example 'show' do + request = create(:submission_request, user:) + + get submission_request_path(request) + + expect(response).to have_http_status(:ok) + + expect(response.parsed_body).to include( + objs: contain_exactly( + include( + _id: '_base' + ) + ), + + validation: nil, + submission: nil + ) + end + + example 'create' do + perform_enqueued_jobs do + post submission_requests_path, params: { + submission_request: { + db: 'JVar', + + objs: [ + { + _id: 'Excel', + file: uploaded_file(name: 'test.xlsx') + } + ] + } + } + end + + expect(response).to have_http_status(:created) + + expect(response.parsed_body).to include( + objs: contain_exactly( + include( + _id: '_base', + validity: 'valid', + ), + include( + _id: 'Excel', + destination: nil, + validity: 'valid', + file_url: a_string_matching(/test\.xlsx$/) + ) + ), + + validation: include( + progress: 'finished', + validity: 'valid', + details: [] + ), + + submission: nil + ) + end +end diff --git a/spec/requests/submissions_spec.rb b/spec/requests/submissions_spec.rb index 3dd59a611..2d58b61a8 100644 --- a/spec/requests/submissions_spec.rb +++ b/spec/requests/submissions_spec.rb @@ -1,165 +1,22 @@ require 'rails_helper' -RSpec.describe '/api/submissions', type: :request, authorized: true do - let_it_be(:user) { create_default(:user, uid: 'alice') } +RSpec.describe '/api/submissions', type: :request do + let(:user) { create(:user) } + let(:default_headers) { {'Authorization' => "Bearer #{user.api_key}"} } - before do - default_headers[:Authorization] = "Bearer #{user.token}" - end - - example 'index' do - create :submission - - get '/api/submissions' - - expect(response).to conform_schema(200) - end - - example 'show' do - travel_to '2024-01-02' - - create :validation, :valid, id: 100, db: 'JVar', created_at: '2024-01-02 03:04:56', started_at: '2024-01-02 03:04:57', finished_at: '2024-01-02 03:04:58' do |validation| - create :submission, validation:, id: 200, created_at: '2024-01-02 03:04:58' - - create :obj, validation:, _id: 'Excel', file: uploaded_file(name: 'myexcel.xlsx'), destination: 'dest', validity: 'valid' - end - - get '/api/submissions/200' - - expect(response).to conform_schema(200) - - expect(response.parsed_body.deep_symbolize_keys).to eq( - id: 200, - url: 'http://www.example.com/api/submissions/200', - db: 'JVar', - created_at: '2024-01-02T03:04:58.000+09:00', - started_at: nil, - finished_at: nil, - progress: 'waiting', - result: nil, - error_message: nil, - accessions: [], - - validation: { - id: 100, - url: 'http://www.example.com/api/validations/100', - - user: { - uid: 'alice' - }, - - db: 'JVar', - created_at: '2024-01-02T03:04:56.000+09:00', - started_at: '2024-01-02T03:04:57.000+09:00', - finished_at: '2024-01-02T03:04:58.000+09:00', - progress: 'finished', - validity: 'valid', - - objects: [ - id: 'Excel', - - files: [ - path: 'dest/myexcel.xlsx', - url: 'http://www.example.com/api/validations/100/files/dest/myexcel.xlsx' - ] - ], - - results: [ - { - object_id: '_base', - validity: 'valid', - details: [], - file: nil - }, - { - object_id: 'Excel', - validity: 'valid', - details: [], - - file: { - path: 'dest/myexcel.xlsx', - url: 'http://www.example.com/api/validations/100/files/dest/myexcel.xlsx' - } - } - ], + example 'create' do + request = create(:submission_request, user:) - raw_result: nil - }, + create :validation, :valid, subject: request - visibility: 'public' - ) - end - - describe 'create' do - example 'ok' do - create :validation, :valid, id: 42, db: 'JVar' - - post '/api/submissions', params: { - db: 'JVar', - validation_id: 42, - visibility: 'public' - }, as: :json - - expect(response).to conform_schema(201) - expect(response.parsed_body.deep_symbolize_keys.dig(:validation, :id)).to eq(42) - end - - example 'validity is not valid' do - create :validation, id: 42, db: 'JVar' - - with_exceptions_app do - post '/api/submissions', params: { - db: 'JVar', - validation_id: 42, - visibility: 'public' - }, as: :json - end - - expect(response).to have_http_status(422) - - expect(response.parsed_body.deep_symbolize_keys).to eq( - error: 'Validation failed: Validation must be valid' - ) + perform_enqueued_jobs do + post submission_request_submission_path(request), params: { + submission: { + visibility: 'public' + } + } end - example 'expired' do - travel_to '2024-01-03 03:04:56' - - create :validation, :valid, id: 42, db: 'JVar', finished_at: '2024-01-02 03:04:56' - - with_exceptions_app do - post '/api/submissions', params: { - db: 'JVar', - validation_id: 42, - visibility: 'public' - }, as: :json - end - - expect(response).to have_http_status(422) - - expect(response.parsed_body.deep_symbolize_keys).to eq( - error: 'Validation failed: Validation finished_at must be in 24 hours' - ) - end - - example 'duplicated' do - create :validation, :valid, id: 42, db: 'JVar' do |validation| - create :submission, validation: - end - - with_exceptions_app do - post '/api/submissions', params: { - db: 'JVar', - validation_id: 42, - visibility: 'public' - }, as: :json - end - - expect(response).to have_http_status(422) - - expect(response.parsed_body.deep_symbolize_keys).to eq( - error: 'Validation failed: Validation is already submitted' - ) - end + expect(response).to have_http_status(:created) end end diff --git a/spec/requests/validations_spec.rb b/spec/requests/validations_spec.rb index 35b9f3400..b6c17bf64 100644 --- a/spec/requests/validations_spec.rb +++ b/spec/requests/validations_spec.rb @@ -11,7 +11,7 @@ before do travel_to '2024-01-02' - create :validation, :valid, id: 100, db: 'GEA', created_at: '2024-01-02 03:04:56', started_at: '2024-01-02 03:04:57', finished_at: '2024-01-02 03:04:58' do |validation| + create :validation, :valid, id: 100, db: 'GEA', created_at: '2024-01-02 03:04:56', finished_at: '2024-01-02 03:04:58' do |validation| create :submission, validation:, id: 200 create :obj, validation:, _id: 'IDF', file: uploaded_file(name: 'myidf.txt'), validity: 'valid' @@ -36,7 +36,6 @@ db: 'MetaboBank', created_at: '2024-01-02T03:04:58.000+09:00', - started_at: nil, finished_at: nil, progress: 'waiting', validity: nil, @@ -64,7 +63,6 @@ db: 'GEA', created_at: '2024-01-02T03:04:56.000+09:00', - started_at: '2024-01-02T03:04:57.000+09:00', finished_at: '2024-01-02T03:04:58.000+09:00', progress: 'finished', validity: 'valid', @@ -104,7 +102,6 @@ url: 'http://www.example.com/api/submissions/200', db: 'GEA', created_at: '2024-01-02T00:00:00.000+09:00', - started_at: nil, finished_at: nil, progress: 'waiting', result: nil, @@ -121,7 +118,7 @@ before do travel_to '2024-01-02' - create :validation, :valid, id: 100, db: 'BioSample', created_at: '2024-01-02 03:04:56', started_at: '2024-01-02 03:04:57', finished_at: '2024-01-02 03:04:58' do |validation| + create :validation, :valid, id: 100, db: 'BioSample', created_at: '2024-01-02 03:04:56', finished_at: '2024-01-02 03:04:58' do |validation| create :submission, validation:, id: 200 end end @@ -141,7 +138,6 @@ db: 'BioSample', created_at: '2024-01-02T03:04:56.000+09:00', - started_at: '2024-01-02T03:04:57.000+09:00', finished_at: '2024-01-02T03:04:58.000+09:00', progress: 'finished', validity: 'valid', @@ -163,7 +159,6 @@ url: 'http://www.example.com/api/submissions/200', db: 'BioSample', created_at: '2024-01-02T00:00:00.000+09:00', - started_at: nil, finished_at: nil, progress: 'waiting', result: nil, diff --git a/web/app/config/environment.ts b/web/app/config/environment.ts index 356792d8a..d7a9b9bef 100644 --- a/web/app/config/environment.ts +++ b/web/app/config/environment.ts @@ -22,4 +22,5 @@ export default config as { rootURL: string; APP: Record; apiURL: string; + directUploadURL: string; } & Record; diff --git a/web/app/router.ts b/web/app/router.ts index 1b686bb2e..82c2e8f6a 100644 --- a/web/app/router.ts +++ b/web/app/router.ts @@ -10,22 +10,23 @@ Router.map(function () { this.route('login'); this.route('account'); - this.route('validations', function () { + this.route('requests', function () { this.route('new'); - this.route('validation', { path: ':validation_id', resetNamespace: true }); + this.route('request', { path: ':request_id', resetNamespace: true }); }); this.route('submissions', function () { this.route('submission', { path: ':submission_id', resetNamespace: true }, function () { - this.route('accession', { path: '/accessions/:number', resetNamespace: true }, function () { - this.route('accession_renewals', { resetNamespace: true }, function () { - this.route('new'); - this.route('accession_renewal', { path: ':accession_renewal_id', resetNamespace: true }); - }); + this.route('updates', { resetNamespace: true }, function () { + this.route('new'); }); }); }); + this.route('updates', function () { + this.route('update', { path: ':update_id', resetNamespace: true }); + }); + this.route('admin', function () { this.route('validations', function () { this.route('validation', { path: ':validation_id' }); diff --git a/web/app/routes/index.ts b/web/app/routes/index.ts index 751dcc2b3..778fb7e70 100644 --- a/web/app/routes/index.ts +++ b/web/app/routes/index.ts @@ -10,7 +10,7 @@ export default class IndexRoute extends Route { beforeModel() { if (this.currentUser.isLoggedIn) { - this.router.transitionTo('validations.index', { queryParams: { page: undefined } }); + this.router.transitionTo('requests', { queryParams: { page: undefined } }); } } } diff --git a/web/app/routes/request.gts b/web/app/routes/request.gts new file mode 100644 index 000000000..2c528664f --- /dev/null +++ b/web/app/routes/request.gts @@ -0,0 +1,14 @@ +import Route from '@ember/routing/route'; +import { service } from '@ember/service'; + +import type RequestService from 'repository/services/request'; + +export default class extends Route { + @service declare request: RequestService; + + async model({ request_id }) { + const res = await this.request.fetchWithModal(`/submission_requests/${request_id}`); + + return res.json(); + } +} diff --git a/web/app/routes/requests.ts b/web/app/routes/requests.ts new file mode 100644 index 000000000..aad928ed1 --- /dev/null +++ b/web/app/routes/requests.ts @@ -0,0 +1,15 @@ +import Route from '@ember/routing/route'; +import { service } from '@ember/service'; + +import type CurrentUserService from 'repository/services/current-user'; +import type RequestService from 'repository/services/request'; +import type Transition from '@ember/routing/transition'; + +export default class SubmissionsRoute extends Route { + @service declare currentUser: CurrentUserService; + @service declare request: RequestService; + + beforeModel(transition: Transition) { + this.currentUser.ensureLogin(transition); + } +} diff --git a/web/app/routes/requests/index.ts b/web/app/routes/requests/index.ts new file mode 100644 index 000000000..f32ba4fa8 --- /dev/null +++ b/web/app/routes/requests/index.ts @@ -0,0 +1,16 @@ +import Route from '@ember/routing/route'; +import { service } from '@ember/service'; + +import ENV from 'repository/config/environment'; + +import type RequestService from 'repository/services/request'; + +export default class extends Route { + @service declare request: RequestService; + + async model() { + const res = await this.request.fetch(`${ENV.apiURL}/submission_requests`); + + return await res.json(); + } +} diff --git a/web/app/routes/update.ts b/web/app/routes/update.ts new file mode 100644 index 000000000..4d59949eb --- /dev/null +++ b/web/app/routes/update.ts @@ -0,0 +1,14 @@ +import Route from '@ember/routing/route'; +import { service } from '@ember/service'; + +import type RequestService from 'repository/services/request'; + +export default class extends Route { + @service declare request: RequestService; + + async model({ update_id }) { + const res = await this.request.fetchWithModal(`/submission_updates/${update_id}`); + + return res.json(); + } +} diff --git a/web/app/templates/application.gts b/web/app/templates/application.gts index a2ee42461..8fe505afd 100644 --- a/web/app/templates/application.gts +++ b/web/app/templates/application.gts @@ -37,11 +37,7 @@ export default class extends Component { {{#if this.currentUser.isLoggedIn}}
  • - Validations -
    • - -
  • - Submissions + Requests
    • {{#if this.currentUser.user.isAdmin}} diff --git a/web/app/templates/request.gts b/web/app/templates/request.gts new file mode 100644 index 000000000..d1d28f72f --- /dev/null +++ b/web/app/templates/request.gts @@ -0,0 +1,111 @@ +import Component from '@glimmer/component'; +import { LinkTo } from '@ember/routing'; +import { action } from '@ember/object'; +import { on } from '@ember/modifier'; +import { service } from '@ember/service'; + +import { eq } from 'ember-truth-helpers'; + +import formatDatetime from 'repository/helpers/format-datetime'; + +import type RequestService from 'ddbj-validate/services/request'; +import type RouterService from '@ember/routing/router-service'; + +export default class extends Component { + @service declare request: RequestService; + @service declare router: RouterService; + + @action + async submit() { + const { model } = this.args; + + await this.request.fetchWithModal(`/submission_requests/${model.id}/submission`, { + method: 'POST', + }); + + this.router.refresh(); + } + + +} diff --git a/web/app/templates/requests/index.gts b/web/app/templates/requests/index.gts new file mode 100644 index 000000000..1f91b4521 --- /dev/null +++ b/web/app/templates/requests/index.gts @@ -0,0 +1,34 @@ +import { LinkTo } from '@ember/routing'; + +import formatDatetime from 'repository/helpers/format-datetime'; + +export default ; diff --git a/web/app/templates/requests/new.gts b/web/app/templates/requests/new.gts new file mode 100644 index 000000000..55ad473ff --- /dev/null +++ b/web/app/templates/requests/new.gts @@ -0,0 +1,75 @@ +import Component from '@glimmer/component'; +import { action } from '@ember/object'; +import { on } from '@ember/modifier'; +import { service } from '@ember/service'; +import { uniqueId } from '@ember/helper'; + +import { DirectUpload } from '@rails/activestorage'; + +import ENV from 'repository/config/environment'; + +import type RequestService from 'repository/services/request'; +import type RouterService from '@ember/routing/router-service'; + +export default class extends Component { + @service declare request: RequestService; + @service declare router: RouterService; + + file?: File; + + @action + selectFile(e: Event) { + this.file = (e.target! as HTMLInputElement).files?.[0]; + } + + @action + submit(e: Event) { + e.preventDefault(); + + if (!this.file) { return; } + + const upload = new DirectUpload(this.file, ENV.directUploadURL); + + upload.create(async (err: Error | null, blob: { signed_id: string }) => { + if (err) { + alert(`Upload failed: ${err.message}`); + return; + } + + const res = await this.request.fetchWithModal('/submission_requests', { + method: 'POST', + + headers: { + 'Content-Type': 'application/json' + }, + + body: JSON.stringify({ + submission_request: { + ddbj_record: blob.signed_id + } + }) + }); + + const { id } = await res.json(); + + this.router.transitionTo('request', id); + }); + } + + +} diff --git a/web/app/templates/submission/index.gts b/web/app/templates/submission/index.gts index 5ed89534f..69a96bbe0 100644 --- a/web/app/templates/submission/index.gts +++ b/web/app/templates/submission/index.gts @@ -1,186 +1,39 @@ -import Component from '@glimmer/component'; import { LinkTo } from '@ember/routing'; -import { action } from '@ember/object'; -import { concat, fn, hash } from '@ember/helper'; -import { getOwner } from '@ember/owner'; -import { on } from '@ember/modifier'; -import { service } from '@ember/service'; -import { eq } from 'ember-truth-helpers'; -import { pageTitle } from 'ember-page-title'; - -import ProgressLabel from 'repository/components/progress-label'; -import ResultBadge from 'repository/components/result-badge'; import formatDatetime from 'repository/helpers/format-datetime'; -import type RequestService from 'repository/services/request'; -import type SubmissionsIndexController from 'repository/controllers/submissions/index'; -import type { components } from 'schema/openapi'; - -type Submission = components['schemas']['SubmissionWithValidation']; - -interface Signature { - Args: { - model: Submission; - }; -} - -export default class extends Component { - @service declare request: RequestService; - - get indexPage() { - const controller = getOwner(this)!.lookup('controller:submissions.index') as SubmissionsIndexController; - - return controller?.pageBefore; - } - - @action - async downloadFile(url: string) { - await this.request.downloadFile(url); - } - - diff --git a/web/app/templates/update.gts b/web/app/templates/update.gts new file mode 100644 index 000000000..db1f51e25 --- /dev/null +++ b/web/app/templates/update.gts @@ -0,0 +1,76 @@ +import { LinkTo } from '@ember/routing'; + +import formatDatetime from 'repository/helpers/format-datetime'; + + diff --git a/web/app/templates/updates/new.gts b/web/app/templates/updates/new.gts new file mode 100644 index 000000000..a943dbdae --- /dev/null +++ b/web/app/templates/updates/new.gts @@ -0,0 +1,75 @@ +import Component from '@glimmer/component'; +import { action } from '@ember/object'; +import { on } from '@ember/modifier'; +import { service } from '@ember/service'; +import { uniqueId } from '@ember/helper'; + +import { DirectUpload } from '@rails/activestorage'; + +import ENV from 'repository/config/environment'; + +import type RequestService from 'repository/services/request'; +import type RouterService from '@ember/routing/router-service'; + +export default class extends Component { + @service declare request: RequestService; + @service declare router: RouterService; + + file?: File; + + @action + selectFile(e: Event) { + this.file = (e.target! as HTMLInputElement).files?.[0]; + } + + @action + submit(e: Event) { + e.preventDefault(); + + if (!this.file) { return; } + + const upload = new DirectUpload(this.file, ENV.directUploadURL); + + upload.create(async (err: Error | null, blob: { signed_id: string }) => { + if (err) { + alert(`Upload failed: ${err.message}`); + return; + } + + const { model } = this.args; + + const res = await this.request.fetchWithModal(`/submissions/${model.id}/updates`, { + method: 'POST', + + headers: { + 'Content-Type': 'application/json' + }, + + body: JSON.stringify({ + submission_update: { + ddbj_record: blob.signed_id + } + }) + }); + + const { submission } = await res.json(); + + this.router.transitionTo('submission', submission.id); + }); + } + + +} diff --git a/web/config/environment.js b/web/config/environment.js index a2cac4d1e..c97b6037e 100644 --- a/web/config/environment.js +++ b/web/config/environment.js @@ -22,6 +22,8 @@ module.exports = function (environment) { apiURL: process.env.API_URL || 'http://repository.localhost:3000/api', }; + ENV.directUploadURL = new URL('/rails/active_storage/direct_uploads', ENV.apiURL).toString(); + if (environment === 'development') { // ENV.APP.LOG_RESOLVER = true; // ENV.APP.LOG_ACTIVE_GENERATION = true; diff --git a/web/package.json b/web/package.json index 9d2362ab2..4db49bc46 100644 --- a/web/package.json +++ b/web/package.json @@ -48,6 +48,7 @@ "@glint/environment-ember-loose": "^1.5.2", "@glint/environment-ember-template-imports": "^1.5.2", "@glint/template": "^1.7.3", + "@rails/activestorage": "^8.1.100", "@rollup/plugin-babel": "^6.1.0", "@tsconfig/ember": "^3.0.12", "@types/bootstrap": "^5.2.10", @@ -101,7 +102,7 @@ "ember": { "edition": "octane" }, - "packageManager": "pnpm@10.22.0+sha512.bf049efe995b28f527fd2b41ae0474ce29186f7edcb3bf545087bd61fbbebb2bf75362d1307fda09c2d288e1e499787ac12d4fcb617a974718a6051f2eee741c", + "packageManager": "pnpm@10.23.0+sha512.21c4e5698002ade97e4efe8b8b4a89a8de3c85a37919f957e7a0f30f38fbc5bbdd05980ffe29179b2fb6e6e691242e098d945d1601772cad0fef5fb6411e2a4b", "ember-addon": { "type": "app", "version": 2 diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index b40dd0887..f32434d7e 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -68,6 +68,9 @@ importers: '@glint/template': specifier: ^1.7.3 version: 1.7.3 + '@rails/activestorage': + specifier: ^8.1.100 + version: 8.1.100 '@rollup/plugin-babel': specifier: ^6.1.0 version: 6.1.0(@babel/core@7.28.5)(rollup@4.53.3) @@ -1370,6 +1373,9 @@ packages: '@popperjs/core@2.11.8': resolution: {integrity: sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==} + '@rails/activestorage@8.1.100': + resolution: {integrity: sha512-rlZg23KYT7JFYXl1tj1WD9nm/yvtUvZT6pzNMVX+HaOjPyGq8xpEv1ADrrhawkbMktKxRj3xDVZ3d/qy0akHvw==} + '@rollup/plugin-babel@6.1.0': resolution: {integrity: sha512-dFZNuFD2YRcoomP4oYf+DvQNSUA9ih+A3vUqopQx5EdtPGo3WBnQcI/S8pwpz91UsGfL0HsMSOlaMld8HrbubA==} engines: {node: '>=14.0.0'} @@ -5620,6 +5626,9 @@ packages: resolution: {integrity: sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==} deprecated: Please use @jridgewell/sourcemap-codec instead + spark-md5@3.0.2: + resolution: {integrity: sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw==} + spawn-args@0.2.0: resolution: {integrity: sha512-73BoniQDcRWgnLAf/suKH6V5H54gd1KLzwYN9FB6J/evqTV33htH9xwV/4BHek+++jzxpVlZQKKZkqstPQPmQg==} @@ -7877,6 +7886,10 @@ snapshots: '@popperjs/core@2.11.8': {} + '@rails/activestorage@8.1.100': + dependencies: + spark-md5: 3.0.2 + '@rollup/plugin-babel@6.1.0(@babel/core@7.28.5)(rollup@4.53.3)': dependencies: '@babel/core': 7.28.5 @@ -7996,6 +8009,27 @@ snapshots: dependencies: '@types/node': 25.0.0 + '@types/ember@4.0.11': + dependencies: + '@types/ember__application': 4.0.11(@babel/core@7.28.5) + '@types/ember__array': 4.0.10(@babel/core@7.28.5) + '@types/ember__component': 4.0.22(@babel/core@7.28.5) + '@types/ember__controller': 4.0.12(@babel/core@7.28.5) + '@types/ember__debug': 4.0.8(@babel/core@7.28.5) + '@types/ember__engine': 4.0.11(@babel/core@7.28.5) + '@types/ember__error': 4.0.6 + '@types/ember__object': 4.0.12(@babel/core@7.28.5) + '@types/ember__polyfills': 4.0.6 + '@types/ember__routing': 4.0.22(@babel/core@7.28.5) + '@types/ember__runloop': 4.0.10 + '@types/ember__service': 4.0.9(@babel/core@7.28.5) + '@types/ember__string': 3.0.15 + '@types/ember__template': 4.0.7 + '@types/ember__test': 4.0.6(@babel/core@7.28.5) + '@types/ember__utils': 4.0.7 + '@types/rsvp': 4.0.9 + optional: true + '@types/ember@4.0.11(@babel/core@7.28.5)': dependencies: '@types/ember__application': 4.0.11(@babel/core@7.28.5) @@ -8023,7 +8057,7 @@ snapshots: '@types/ember__application@4.0.11(@babel/core@7.28.5)': dependencies: '@glimmer/component': 1.1.2(@babel/core@7.28.5) - '@types/ember': 4.0.11(@babel/core@7.28.5) + '@types/ember': 4.0.11 '@types/ember__engine': 4.0.11(@babel/core@7.28.5) '@types/ember__object': 4.0.12(@babel/core@7.28.5) '@types/ember__owner': 4.0.9 @@ -8125,6 +8159,11 @@ snapshots: - supports-color optional: true + '@types/ember__runloop@4.0.10': + dependencies: + '@types/ember': 4.0.11 + optional: true + '@types/ember__runloop@4.0.10(@babel/core@7.28.5)': dependencies: '@types/ember': 4.0.11(@babel/core@7.28.5) @@ -8155,6 +8194,11 @@ snapshots: - supports-color optional: true + '@types/ember__utils@4.0.7': + dependencies: + '@types/ember': 4.0.11 + optional: true + '@types/ember__utils@4.0.7(@babel/core@7.28.5)': dependencies: '@types/ember': 4.0.11(@babel/core@7.28.5) @@ -13155,6 +13199,8 @@ snapshots: sourcemap-codec@1.4.8: {} + spark-md5@3.0.2: {} + spawn-args@0.2.0: {} split-string@3.1.0: From 241fec57629ebb30a6b8c43d3a00577582fe5c45 Mon Sep 17 00:00:00 2001 From: Keita Urashima Date: Tue, 9 Dec 2025 15:42:46 +0900 Subject: [PATCH 2/5] Update changed entries --- app/jobs/apply_submission_update_job.rb | 24 ++++++++++++++++++----- app/models/concerns/validation_subject.rb | 3 ++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/app/jobs/apply_submission_update_job.rb b/app/jobs/apply_submission_update_job.rb index 46993a0ab..ed1e9e6d2 100644 --- a/app/jobs/apply_submission_update_job.rb +++ b/app/jobs/apply_submission_update_job.rb @@ -1,13 +1,17 @@ class ApplySubmissionUpdateJob < ApplicationJob + class NoChange < StandardError; end + def perform(update) DDBJRecordValidator.validate update return unless update.ready_to_apply? - begin - update.applying! + update.applying! + begin apply update + rescue NoChange + update.no_change! rescue => e Rails.error.report e @@ -23,14 +27,24 @@ def perform(update) private def apply(update) - record = JSON.parse(update.ddbj_record.download, symbolize_names: true) - entries = record.dig(:sequence, :entries) + old_record = JSON.parse(update.submission.ddbj_record.download, symbolize_names: true) + new_record = JSON.parse(update.ddbj_record.download, symbolize_names: true) + old_entries = old_record.dig(:sequence, :entries).index_by { it[:accession] } + new_entries = new_record.dig(:sequence, :entries) + + changes = new_entries.select {|new_entry| + old_entry = old_entries.fetch(new_entry[:accession]) + + new_entry != old_entry + } + + raise NoChange if changes.empty? ActiveRecord::Base.transaction do number_to_accession = update.submission.accessions.index_by(&:number) now = Time.current - accession_to_attrs = update.submission.accessions.upsert_all(entries.map {|entry| + accession_to_attrs = update.submission.accessions.upsert_all(changes.map {|entry| acc = number_to_accession.fetch(entry[:accession]) { diff --git a/app/models/concerns/validation_subject.rb b/app/models/concerns/validation_subject.rb index 10e6660c7..362bdaaa5 100644 --- a/app/models/concerns/validation_subject.rb +++ b/app/models/concerns/validation_subject.rb @@ -12,7 +12,8 @@ module ValidationSubject ready_to_apply: 3, applying: 4, applied: 5, - application_failed: 6 + application_failed: 6, + no_change: 7 } scope :with_validity, -> { From b114d1304ac8afec2255973a0b97dd575879ab60 Mon Sep 17 00:00:00 2001 From: maimux2x Date: Thu, 11 Dec 2025 16:24:25 +0900 Subject: [PATCH 3/5] Update submission after confirmation --- .../submission_requests_controller.rb | 2 +- .../submission_updates_controller.rb | 2 +- app/controllers/submissions_controller.rb | 14 ++ app/jobs/apply_submission_request_job.rb | 4 +- app/jobs/apply_submission_update_job.rb | 3 +- app/jobs/validate_ddbj_record_job.rb | 5 + app/jobs/validate_submission_request_job.rb | 5 - config/routes.rb | 4 +- web/app/router.ts | 4 + web/app/templates/request.gts | 13 +- web/app/templates/update.gts | 153 +++++++++++------- web/app/templates/updates/new.gts | 6 +- 12 files changed, 143 insertions(+), 72 deletions(-) create mode 100644 app/jobs/validate_ddbj_record_job.rb delete mode 100644 app/jobs/validate_submission_request_job.rb diff --git a/app/controllers/submission_requests_controller.rb b/app/controllers/submission_requests_controller.rb index b0a13c15f..7a83cf7f6 100644 --- a/app/controllers/submission_requests_controller.rb +++ b/app/controllers/submission_requests_controller.rb @@ -17,7 +17,7 @@ def show def create @request = current_user.submission_requests.create!(request_params) - ValidateSubmissionRequestJob.perform_later @request + ValidateDDBJRecordJob.perform_later @request render :show, status: :accepted end diff --git a/app/controllers/submission_updates_controller.rb b/app/controllers/submission_updates_controller.rb index 9c6f0f57f..44ef9eee3 100644 --- a/app/controllers/submission_updates_controller.rb +++ b/app/controllers/submission_updates_controller.rb @@ -8,7 +8,7 @@ def create @update = submission.updates.create!(update_params) - ApplySubmissionUpdateJob.perform_later @update + ValidateDDBJRecordJob.perform_later @update render :show, status: :accepted end diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index fab7141a2..174d1b17f 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -20,4 +20,18 @@ def create head :accepted end + + def update + update = current_user.submission_updates.valid_only.joins( + :validation + ).where( + validations: { + finished_at: 1.day.ago.. + } + ).find(params[:submission_update_id]) + + ApplySubmissionUpdateJob.perform_later update + + head :accepted + end end diff --git a/app/jobs/apply_submission_request_job.rb b/app/jobs/apply_submission_request_job.rb index 554b418c5..5b466ba55 100644 --- a/app/jobs/apply_submission_request_job.rb +++ b/app/jobs/apply_submission_request_job.rb @@ -1,5 +1,7 @@ class ApplySubmissionRequestJob < ApplicationJob def perform(request) + return unless request.ready_to_apply? + ActiveRecord::Base.transaction do request.applying! request.create_submission! @@ -16,7 +18,7 @@ def perform(request) ) else request.applied! - request.validation.write_submission_files to: submission.dir + request.validation.write_submission_files to: request.submission.dir end end diff --git a/app/jobs/apply_submission_update_job.rb b/app/jobs/apply_submission_update_job.rb index ed1e9e6d2..a2c6bb66d 100644 --- a/app/jobs/apply_submission_update_job.rb +++ b/app/jobs/apply_submission_update_job.rb @@ -2,8 +2,6 @@ class ApplySubmissionUpdateJob < ApplicationJob class NoChange < StandardError; end def perform(update) - DDBJRecordValidator.validate update - return unless update.ready_to_apply? update.applying! @@ -21,6 +19,7 @@ def perform(update) ) else update.applied! + update.validation.write_submission_files to: update.submission.dir end end diff --git a/app/jobs/validate_ddbj_record_job.rb b/app/jobs/validate_ddbj_record_job.rb new file mode 100644 index 000000000..0ca928bcb --- /dev/null +++ b/app/jobs/validate_ddbj_record_job.rb @@ -0,0 +1,5 @@ +class ValidateDDBJRecordJob < ApplicationJob + def perform(subject) + DDBJRecordValidator.validate subject + end +end diff --git a/app/jobs/validate_submission_request_job.rb b/app/jobs/validate_submission_request_job.rb deleted file mode 100644 index ab2c4e116..000000000 --- a/app/jobs/validate_submission_request_job.rb +++ /dev/null @@ -1,5 +0,0 @@ -class ValidateSubmissionRequestJob < ApplicationJob - def perform(request) - DDBJRecordValidator.validate request - end -end diff --git a/config/routes.rb b/config/routes.rb index d18624a00..1f6b5fcd8 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -15,7 +15,9 @@ resource :submission, only: :create end - resources :submission_updates, only: %i[show] + resources :submission_updates, only: %i[show] do + resource :submission, only: :update + end resources :submissions, only: %i[index show] do resources :updates, only: %i[create], controller: 'submission_updates' diff --git a/web/app/router.ts b/web/app/router.ts index 82c2e8f6a..8991228a2 100644 --- a/web/app/router.ts +++ b/web/app/router.ts @@ -15,6 +15,10 @@ Router.map(function () { this.route('request', { path: ':request_id', resetNamespace: true }); }); + this.route('updates', function() { + this.route('update', { path: ':update_id', resetNamespace: true }); + }); + this.route('submissions', function () { this.route('submission', { path: ':submission_id', resetNamespace: true }, function () { this.route('updates', { resetNamespace: true }, function () { diff --git a/web/app/templates/request.gts b/web/app/templates/request.gts index d1d28f72f..4645ecce9 100644 --- a/web/app/templates/request.gts +++ b/web/app/templates/request.gts @@ -16,7 +16,7 @@ export default class extends Component { @service declare router: RouterService; @action - async submit() { + async apply() { const { model } = this.args; await this.request.fetchWithModal(`/submission_requests/${model.id}/submission`, { @@ -58,7 +58,14 @@ export default class extends Component {
    {{formatDatetime @model.validation.created_at}}
    Finished
    -
    {{formatDatetime @model.validation.finished_at}}
    + +
    + {{#if @model.validation.finished_at}} + {{formatDatetime @model.validation.finished_at}} + {{else}} + - + {{/if}} +
    Validity
    {{@model.validation.validity}}
    @@ -94,7 +101,7 @@ export default class extends Component { {{#if (eq @model.status "ready_to_apply")}}
    -
    diff --git a/web/app/templates/update.gts b/web/app/templates/update.gts index db1f51e25..e6a5e3feb 100644 --- a/web/app/templates/update.gts +++ b/web/app/templates/update.gts @@ -1,76 +1,119 @@ +import Component from '@glimmer/component'; import { LinkTo } from '@ember/routing'; +import { action } from '@ember/object'; +import { on } from '@ember/modifier'; +import { service } from '@ember/service'; + +import { eq } from 'ember-truth-helpers'; import formatDatetime from 'repository/helpers/format-datetime'; - +} diff --git a/web/app/templates/updates/new.gts b/web/app/templates/updates/new.gts index a943dbdae..4237fc8f5 100644 --- a/web/app/templates/updates/new.gts +++ b/web/app/templates/updates/new.gts @@ -52,9 +52,9 @@ export default class extends Component { }) }); - const { submission } = await res.json(); + const { id } = await res.json(); - this.router.transitionTo('submission', submission.id); + this.router.transitionTo('update', id); }); } @@ -69,7 +69,7 @@ export default class extends Component { {{/let}} - + } From 87c1349710da318f8987656e11bf5ca65fb98758 Mon Sep 17 00:00:00 2001 From: maimux2x Date: Thu, 11 Dec 2025 17:47:39 +0900 Subject: [PATCH 4/5] Add submissions index --- app/controllers/submissions_controller.rb | 4 + app/views/submissions/_submission.json.jb | 3 +- web/app/router.ts | 6 +- web/app/routes/submissions/index.ts | 85 +------------ web/app/templates/application.gts | 4 + web/app/templates/submission/index.gts | 8 +- .../{ => submission}/updates/new.gts | 0 web/app/templates/submissions/index.gts | 117 ++++-------------- 8 files changed, 46 insertions(+), 181 deletions(-) rename web/app/templates/{ => submission}/updates/new.gts (100%) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 174d1b17f..c46546080 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -1,4 +1,8 @@ class SubmissionsController < ApplicationController + def index + @submissions = current_user.submissions + end + def show @submission = current_user.submissions.includes( :updates diff --git a/app/views/submissions/_submission.json.jb b/app/views/submissions/_submission.json.jb index 1e7d3327d..10821045d 100644 --- a/app/views/submissions/_submission.json.jb +++ b/app/views/submissions/_submission.json.jb @@ -3,7 +3,8 @@ { **submission.slice( :id, - :created_at + :created_at, + :updated_at ), ddbj_record: submission.ddbj_record.then {|record| diff --git a/web/app/router.ts b/web/app/router.ts index 8991228a2..0746de309 100644 --- a/web/app/router.ts +++ b/web/app/router.ts @@ -21,16 +21,12 @@ Router.map(function () { this.route('submissions', function () { this.route('submission', { path: ':submission_id', resetNamespace: true }, function () { - this.route('updates', { resetNamespace: true }, function () { + this.route('updates', function () { this.route('new'); }); }); }); - this.route('updates', function () { - this.route('update', { path: ':update_id', resetNamespace: true }); - }); - this.route('admin', function () { this.route('validations', function () { this.route('validation', { path: ':validation_id' }); diff --git a/web/app/routes/submissions/index.ts b/web/app/routes/submissions/index.ts index bec3051ae..0fe0bb2d9 100644 --- a/web/app/routes/submissions/index.ts +++ b/web/app/routes/submissions/index.ts @@ -2,92 +2,15 @@ import Route from '@ember/routing/route'; import { service } from '@ember/service'; import ENV from 'repository/config/environment'; -import convertCreatedToDate from 'repository/utils/convert-created-to-date'; import type RequestService from 'repository/services/request'; -import type SubmissionsIndexController from 'repository/controllers/submissions'; -import type { Created } from 'repository/models/criteria'; -import type { components } from 'schema/openapi'; -type Submission = components['schemas']['SubmissionWithValidation']; - -export interface Model { - submissions: Submission[]; - totalPages: number; -} - -interface Params { - page?: number; - db?: string; - created?: Created; - result?: string; -} - -export default class SubmissionsIndexRoute extends Route { +export default class extends Route { @service declare request: RequestService; - timer?: number; - - queryParams = { - page: { - refreshModel: true, - }, - db: { - refreshModel: true, - }, - created: { - refreshModel: true, - }, - result: { - refreshModel: true, - }, - }; - - async model(params: Params) { - const url = new URL(`${ENV.apiURL}/submissions`); - - if (params.page !== undefined) { - url.searchParams.set('page', params.page.toString()); - } - - if (params.db !== undefined) { - url.searchParams.set('db', params.db); - } - - if (params.created !== undefined) { - url.searchParams.set('created_at_after', convertCreatedToDate(params.created).toISOString()); - } - - if (params.result !== undefined) { - url.searchParams.set('result', params.result); - } - - const res = await this.request.fetch(url.toString()); - - return { - submissions: (await res.json()) as Submission[], - totalPages: parseInt(res.headers.get('Total-Pages')!), - }; - } - - afterModel(model: Model) { - if (model.submissions.some(({ progress }) => progress === 'waiting' || progress === 'running')) { - this.timer = setTimeout(() => { - this.refresh(); - }, 2000); - } - } - - resetController(controller: SubmissionsIndexController, isExiting: boolean) { - if (isExiting) { - controller.pageBefore = controller.page; - controller.page = 1; - } - } + async model() { + const res = await this.request.fetch(`${ENV.apiURL}/submissions`); - deactivate() { - if (this.timer) { - clearTimeout(this.timer); - } + return await res.json(); } } diff --git a/web/app/templates/application.gts b/web/app/templates/application.gts index 8fe505afd..dab49422a 100644 --- a/web/app/templates/application.gts +++ b/web/app/templates/application.gts @@ -40,6 +40,10 @@ export default class extends Component { Requests +
  • + Submissions +
    • + {{#if this.currentUser.user.isAdmin}}
  • Administration diff --git a/web/app/templates/submission/index.gts b/web/app/templates/submission/index.gts index 69a96bbe0..376811b32 100644 --- a/web/app/templates/submission/index.gts +++ b/web/app/templates/submission/index.gts @@ -6,6 +6,12 @@ import formatDatetime from 'repository/helpers/format-datetime';

    Submission-{{@model.id}}

    +
    Created
    +
    {{formatDatetime @model.created_at}}
    + +
    Updated
    +
    {{formatDatetime @model.updated_at}}
    +
    File
    @@ -46,7 +52,7 @@ import formatDatetime from 'repository/helpers/format-datetime';

    Updates

    - Update Submission + Update Submission
    diff --git a/web/app/templates/updates/new.gts b/web/app/templates/submission/updates/new.gts similarity index 100% rename from web/app/templates/updates/new.gts rename to web/app/templates/submission/updates/new.gts diff --git a/web/app/templates/submissions/index.gts b/web/app/templates/submissions/index.gts index ff3ef9709..4c9e3fcfe 100644 --- a/web/app/templates/submissions/index.gts +++ b/web/app/templates/submissions/index.gts @@ -1,101 +1,32 @@ -import Component from '@glimmer/component'; import { LinkTo } from '@ember/routing'; -import { action } from '@ember/object'; -import { or, notEq } from 'ember-truth-helpers'; - -import Pagination from 'repository/components/pagination'; -import ProgressLabel from 'repository/components/progress-label'; -import ResultBadge from 'repository/components/result-badge'; -import SubmissionsSearchForm from 'repository/components/submissions-search-form'; -import Table from 'repository/components/table'; import formatDatetime from 'repository/helpers/format-datetime'; -import type SubmissionsIndexController from 'repository/controllers/submissions/index'; -import type { Model } from 'repository/routes/submissions/index'; -import type { Query } from 'repository/components/submissions-search-form'; - -interface Signature { - Args: { - model: Model; - controller: SubmissionsIndexController; - }; -} - -export default class extends Component { - @action - applyQuery(query: Query) { - const { controller } = this.args; - - Object.assign(controller, query, { page: 1 }); - } +export default From 79c0080ad3c6bdb2ec3a2e30a6893a2be8d2696f Mon Sep 17 00:00:00 2001 From: Keita Urashima Date: Fri, 12 Dec 2025 16:32:42 +0900 Subject: [PATCH 5/5] Style --- web/app/templates/request.gts | 2 +- web/app/templates/requests/index.gts | 10 ++++++---- web/app/templates/requests/new.gts | 2 +- web/app/templates/submission/index.gts | 10 +++++----- web/app/templates/submission/updates/new.gts | 2 +- web/app/templates/submissions/index.gts | 6 +++--- web/app/templates/update.gts | 2 +- 7 files changed, 18 insertions(+), 16 deletions(-) diff --git a/web/app/templates/request.gts b/web/app/templates/request.gts index 4645ecce9..1dcf782e5 100644 --- a/web/app/templates/request.gts +++ b/web/app/templates/request.gts @@ -27,7 +27,7 @@ export default class extends Component { }