diff --git a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt index 513e542dca..154b9f2ffc 100644 --- a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt +++ b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt @@ -23,7 +23,7 @@ data class Schema( val metadata: List, val externalMetadata: List = emptyList(), val earliestReleaseDate: EarliestReleaseDate = EarliestReleaseDate(false, emptyList()), - val allowSubmissionOfConsensusSequences: Boolean + val allowSubmissionOfConsensusSequences: Boolean = true, ) // The Json property names need to be kept in sync with website config enum `metadataPossibleTypes` in `config.ts` diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt index 7420815501..88ba74b6ad 100644 --- a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt +++ b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt @@ -86,7 +86,7 @@ open class SubmissionController( @HiddenParam authenticatedUser: AuthenticatedUser, @Parameter(description = GROUP_ID_DESCRIPTION) @RequestParam groupId: Int, @Parameter(description = METADATA_FILE_DESCRIPTION) @RequestParam metadataFile: MultipartFile, - @Parameter(description = SEQUENCE_FILE_DESCRIPTION) @RequestParam sequenceFile: MultipartFile, + @Parameter(description = SEQUENCE_FILE_DESCRIPTION) @RequestParam sequenceFile: MultipartFile?, @Parameter(description = "Data Use terms under which data is released.") @RequestParam dataUseTermsType: DataUseTermsType, @Parameter( @@ -118,7 +118,7 @@ open class SubmissionController( ) @RequestParam metadataFile: MultipartFile, @Parameter( description = SEQUENCE_FILE_DESCRIPTION, - ) @RequestParam sequenceFile: MultipartFile, + ) @RequestParam sequenceFile: MultipartFile?, ): List { val params = SubmissionParams.RevisionSubmissionParams( organism, @@ -172,7 +172,9 @@ open class SubmissionController( } val lastDatabaseWriteETag = releasedDataModel.getLastDatabaseWriteETag() - if (ifNoneMatch == lastDatabaseWriteETag) return ResponseEntity.status(HttpStatus.NOT_MODIFIED).build() + if (ifNoneMatch == lastDatabaseWriteETag) { + return ResponseEntity.status(HttpStatus.NOT_MODIFIED).build() + } val headers = HttpHeaders() headers.contentType = MediaType.parseMediaType(MediaType.APPLICATION_NDJSON_VALUE) diff --git a/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt index 3c17f0157e..1f26b9de95 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt @@ -8,6 +8,7 @@ import org.loculus.backend.api.DataUseTerms import org.loculus.backend.api.Organism import org.loculus.backend.api.SubmissionIdMapping import org.loculus.backend.auth.AuthenticatedUser +import org.loculus.backend.config.BackendConfig import org.loculus.backend.controller.BadRequestException import org.loculus.backend.controller.DuplicateKeyException import org.loculus.backend.controller.UnprocessableEntityException @@ -41,14 +42,14 @@ interface SubmissionParams { val organism: Organism val authenticatedUser: AuthenticatedUser val metadataFile: MultipartFile - val sequenceFile: MultipartFile + val sequenceFile: MultipartFile? val uploadType: UploadType data class OriginalSubmissionParams( override val organism: Organism, override val authenticatedUser: AuthenticatedUser, override val metadataFile: MultipartFile, - override val sequenceFile: MultipartFile, + override val sequenceFile: MultipartFile?, val groupId: Int, val dataUseTerms: DataUseTerms, ) : SubmissionParams { @@ -59,7 +60,7 @@ interface SubmissionParams { override val organism: Organism, override val authenticatedUser: AuthenticatedUser, override val metadataFile: MultipartFile, - override val sequenceFile: MultipartFile, + override val sequenceFile: MultipartFile?, ) : SubmissionParams { override val uploadType: UploadType = UploadType.REVISION } @@ -76,6 +77,7 @@ class SubmitModel( private val groupManagementPreconditionValidator: GroupManagementPreconditionValidator, private val dataUseTermsPreconditionValidator: DataUseTermsPreconditionValidator, private val dateProvider: DateProvider, + private val backendConfig: BackendConfig, ) { companion object AcceptedFileTypes { @@ -106,9 +108,11 @@ class SubmitModel( batchSize, ) - log.debug { "Validating submission with uploadId $uploadId" } - val (metadataSubmissionIds, sequencesSubmissionIds) = uploadDatabaseService.getUploadSubmissionIds(uploadId) - validateSubmissionIdSets(metadataSubmissionIds.toSet(), sequencesSubmissionIds.toSet()) + if (requiresSequenceFile(submissionParams.organism)) { + log.debug { "Validating submission with uploadId $uploadId" } + val (metadataSubmissionIds, sequencesSubmissionIds) = uploadDatabaseService.getUploadSubmissionIds(uploadId) + validateSubmissionIdSets(metadataSubmissionIds.toSet(), sequencesSubmissionIds.toSet()) + } if (submissionParams is SubmissionParams.RevisionSubmissionParams) { log.info { "Associating uploaded sequence data with existing sequence entries with uploadId $uploadId" } @@ -150,17 +154,32 @@ class SubmitModel( metadataTempFileToDelete.delete() } - val sequenceTempFileToDelete = MaybeFile() - try { - val sequenceStream = getStreamFromFile( - submissionParams.sequenceFile, - uploadId, - sequenceFileTypes, - sequenceTempFileToDelete, - ) - uploadSequences(uploadId, sequenceStream, batchSize, submissionParams.organism) - } finally { - sequenceTempFileToDelete.delete() + val sequenceFile = submissionParams.sequenceFile + if (sequenceFile == null) { + if (requiresSequenceFile(submissionParams.organism)) { + throw BadRequestException( + "Submissions for organism ${submissionParams.organism.name} require a sequence file.", + ) + } + } else { + if (!requiresSequenceFile(submissionParams.organism)) { + throw BadRequestException( + "Sequence uploads are not allowed for organism ${submissionParams.organism.name}.", + ) + } + + val sequenceTempFileToDelete = MaybeFile() + try { + val sequenceStream = getStreamFromFile( + sequenceFile, + uploadId, + sequenceFileTypes, + sequenceTempFileToDelete, + ) + uploadSequences(uploadId, sequenceStream, batchSize, submissionParams.organism) + } finally { + sequenceTempFileToDelete.delete() + } } } @@ -324,4 +343,8 @@ class SubmitModel( SequenceUploadAuxTable.select(SequenceUploadAuxTable.sequenceSubmissionIdColumn).count() > 0 return metadataInAuxTable || sequencesInAuxTable } + + private fun requiresSequenceFile(organism: Organism) = backendConfig.getInstanceConfig(organism) + .schema + .allowSubmissionOfConsensusSequences } diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt index 17cff685b4..900ce64ead 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt @@ -149,14 +149,17 @@ class UploadDatabaseService( jsonb_build_object( 'metadata', metadata_upload_aux_table.metadata, 'unalignedNucleotideSequences', - jsonb_object_agg( - sequence_upload_aux_table.segment_name, - sequence_upload_aux_table.compressed_sequence_data::jsonb + COALESCE( + jsonb_object_agg( + sequence_upload_aux_table.segment_name, + sequence_upload_aux_table.compressed_sequence_data::jsonb + ) FILTER (WHERE sequence_upload_aux_table.segment_name IS NOT NULL), + '{}'::jsonb ) ) FROM metadata_upload_aux_table - JOIN + LEFT JOIN sequence_upload_aux_table ON metadata_upload_aux_table.upload_id = sequence_upload_aux_table.upload_id AND metadata_upload_aux_table.submission_id = sequence_upload_aux_table.submission_id diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt b/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt index 6cf573c562..a3289d39a4 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt @@ -31,6 +31,7 @@ import org.testcontainers.shaded.org.awaitility.Awaitility.await const val DEFAULT_ORGANISM = "dummyOrganism" const val OTHER_ORGANISM = "otherOrganism" +const val ORGANISM_WITHOUT_SEQUENCES = "dummyOrganismWithoutSequences" const val DEFAULT_PIPELINE_VERSION = 1L const val DEFAULT_EXTERNAL_METADATA_UPDATER = "ena" diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/ExtractUnprocessedDataEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/ExtractUnprocessedDataEndpointTest.kt index 52bbfa62ce..cb8179c1bf 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/ExtractUnprocessedDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/ExtractUnprocessedDataEndpointTest.kt @@ -5,6 +5,7 @@ import org.hamcrest.CoreMatchers.hasItem import org.hamcrest.CoreMatchers.`is` import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.allOf +import org.hamcrest.Matchers.anEmptyMap import org.hamcrest.Matchers.containsInAnyOrder import org.hamcrest.Matchers.empty import org.hamcrest.Matchers.greaterThan @@ -12,6 +13,8 @@ import org.hamcrest.Matchers.hasProperty import org.hamcrest.Matchers.hasSize import org.hamcrest.Matchers.matchesRegex import org.junit.jupiter.api.Test +import org.loculus.backend.api.GeneticSequence +import org.loculus.backend.api.OriginalData import org.loculus.backend.api.Status.IN_PROCESSING import org.loculus.backend.api.Status.RECEIVED import org.loculus.backend.api.UnprocessedData @@ -19,6 +22,7 @@ import org.loculus.backend.config.BackendSpringProperty import org.loculus.backend.controller.DEFAULT_ORGANISM import org.loculus.backend.controller.DEFAULT_USER_NAME import org.loculus.backend.controller.EndpointTest +import org.loculus.backend.controller.ORGANISM_WITHOUT_SEQUENCES import org.loculus.backend.controller.OTHER_ORGANISM import org.loculus.backend.controller.assertStatusIs import org.loculus.backend.controller.expectForbiddenResponse @@ -27,7 +31,6 @@ import org.loculus.backend.controller.expectUnauthorizedResponse import org.loculus.backend.controller.getAccessionVersions import org.loculus.backend.controller.jwtForDefaultUser import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles -import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.HttpHeaders.ETAG import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header @@ -181,4 +184,37 @@ class ExtractUnprocessedDataEndpointTest( `is`(empty()), ) } + + @Test + fun `GIVEN sequences for organism without sequences THEN only returns metadata`() { + val submissionResult = convenienceClient.submitDefaultFiles(organism = ORGANISM_WITHOUT_SEQUENCES) + val accessionVersions = submissionResult.submissionIdMappings + + val result = client.extractUnprocessedData( + numberOfSequenceEntries = DefaultFiles.NUMBER_OF_SEQUENCES, + organism = ORGANISM_WITHOUT_SEQUENCES, + ) + val responseBody = result.expectNdjsonAndGetContent() + assertThat(responseBody, hasSize(DefaultFiles.NUMBER_OF_SEQUENCES)) + assertThat( + responseBody, + hasItem( + allOf( + hasProperty("accession", `is`(accessionVersions[0].accession)), + hasProperty("version", `is`(1L)), + hasProperty( + "data", + allOf( + hasProperty>("metadata", `is`(defaultOriginalData.metadata)), + hasProperty("unalignedNucleotideSequences", `is`(anEmptyMap())), + ), + ), + hasProperty("submissionId", matchesRegex("custom[0-9]")), + hasProperty("submitter", `is`(DEFAULT_USER_NAME)), + hasProperty("groupId", `is`(submissionResult.groupId)), + hasProperty("submittedAt", greaterThan(1_700_000_000L)), + ), + ), + ) + } } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/PreparedProcessedData.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/PreparedProcessedData.kt index f3446306ed..b887f1a08a 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/PreparedProcessedData.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/PreparedProcessedData.kt @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.IntNode import com.fasterxml.jackson.databind.node.NullNode import com.fasterxml.jackson.databind.node.TextNode import org.loculus.backend.api.GeneName +import org.loculus.backend.api.GeneticSequence import org.loculus.backend.api.Insertion import org.loculus.backend.api.PreprocessingAnnotation import org.loculus.backend.api.PreprocessingAnnotationSource @@ -99,6 +100,21 @@ val defaultProcessedDataMultiSegmented = ProcessedData( ), ) +val defaultProcessedDataWithoutSequences = ProcessedData( + metadata = mapOf( + "date" to TextNode("2002-12-15"), + "host" to TextNode("google.com"), + "region" to TextNode("Europe"), + "country" to TextNode("Spain"), + "division" to NullNode.instance, + ), + unalignedNucleotideSequences = emptyMap(), + alignedNucleotideSequences = emptyMap(), + nucleotideInsertions = emptyMap(), + alignedAminoAcidSequences = emptyMap(), + aminoAcidInsertions = emptyMap(), +) + private val defaultSuccessfulSubmittedData = SubmittedProcessedData( accession = "If a test result shows this, processed data was not prepared correctly.", version = 1, diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/ReviseEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/ReviseEndpointTest.kt index 4d9c361c3d..1ae336fa06 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/ReviseEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/ReviseEndpointTest.kt @@ -18,6 +18,7 @@ import org.loculus.backend.api.UnprocessedData import org.loculus.backend.controller.DEFAULT_ORGANISM import org.loculus.backend.controller.DEFAULT_USER_NAME import org.loculus.backend.controller.EndpointTest +import org.loculus.backend.controller.ORGANISM_WITHOUT_SEQUENCES import org.loculus.backend.controller.OTHER_ORGANISM import org.loculus.backend.controller.SUPER_USER_NAME import org.loculus.backend.controller.assertStatusIs @@ -29,7 +30,7 @@ import org.loculus.backend.controller.groupmanagement.andGetGroupId import org.loculus.backend.controller.jwtForSuperUser import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles import org.springframework.beans.factory.annotation.Autowired -import org.springframework.http.MediaType +import org.springframework.http.MediaType.APPLICATION_JSON_VALUE import org.springframework.mock.web.MockMultipartFile import org.springframework.test.web.servlet.ResultMatcher import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content @@ -67,7 +68,7 @@ class ReviseEndpointTest( jwt = jwtForSuperUser, ) .andExpect(status().isOk) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) .andExpect(jsonPath("\$.length()").value(DefaultFiles.NUMBER_OF_SEQUENCES)) .andExpect(jsonPath("\$[0].submissionId").value("custom0")) .andExpect(jsonPath("\$[0].accession").value(accessions.first())) @@ -86,7 +87,7 @@ class ReviseEndpointTest( DefaultFiles.sequencesFile, ) .andExpect(status().isOk) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) .andExpect(jsonPath("\$.length()").value(DefaultFiles.NUMBER_OF_SEQUENCES)) .andExpect(jsonPath("\$[0].submissionId").value("custom0")) .andExpect(jsonPath("\$[0].accession").value(accessions.first())) @@ -129,7 +130,7 @@ class ReviseEndpointTest( ), SubmitFiles.sequenceFileWith(), ).andExpect(status().isUnprocessableEntity) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) .andExpect( jsonPath("\$.detail").value( "Accessions 123 do not exist", @@ -149,7 +150,7 @@ class ReviseEndpointTest( organism = OTHER_ORGANISM, ) .andExpect(status().isUnprocessableEntity) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) .andExpect( jsonPath("\$.detail").value( containsString("accession versions are not of organism otherOrganism:"), @@ -168,7 +169,7 @@ class ReviseEndpointTest( jwt = generateJwtFor(notSubmitter), ) .andExpect(status().isForbidden) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) .andExpect( jsonPath( "\$.detail", @@ -186,7 +187,7 @@ class ReviseEndpointTest( DefaultFiles.sequencesFile, ) .andExpect(status().isUnprocessableEntity) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) .andExpect( jsonPath( "\$.detail", @@ -198,6 +199,64 @@ class ReviseEndpointTest( ) } + @Test + fun `GIVEN no sequences file for organism that requires one THEN throws bad request error`() { + val accessions = convenienceClient.prepareDataTo(APPROVED_FOR_RELEASE).map { it.accession } + + client.reviseSequenceEntries( + metadataFile = DefaultFiles.getRevisedMetadataFile(accessions), + sequencesFile = null, + ) + .andExpect(status().isBadRequest) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) + .andExpect( + jsonPath("\$.detail").value("Submissions for organism $DEFAULT_ORGANISM require a sequence file."), + ) + } + + @Test + fun `GIVEN sequence file for organism without sequences THEN returns bad request`() { + val accessions = convenienceClient.prepareDataTo( + status = APPROVED_FOR_RELEASE, + organism = ORGANISM_WITHOUT_SEQUENCES, + ) + .map { it.accession } + + client.reviseSequenceEntries( + DefaultFiles.getRevisedMetadataFile(accessions), + DefaultFiles.sequencesFile, + organism = ORGANISM_WITHOUT_SEQUENCES, + ) + .andExpect(status().isBadRequest) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) + .andExpect( + jsonPath( + "\$.detail", + ).value("Sequence uploads are not allowed for organism $ORGANISM_WITHOUT_SEQUENCES."), + ) + } + + @Test + fun `GIVEN no sequence file for organism without sequences THEN data is accepted`() { + val accessions = convenienceClient.prepareDataTo( + status = APPROVED_FOR_RELEASE, + organism = ORGANISM_WITHOUT_SEQUENCES, + ) + .map { it.accession } + + client.reviseSequenceEntries( + metadataFile = DefaultFiles.getRevisedMetadataFile(accessions), + sequencesFile = null, + organism = ORGANISM_WITHOUT_SEQUENCES, + ) + .andExpect(status().isOk) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) + .andExpect(jsonPath("\$.length()").value(DefaultFiles.NUMBER_OF_SEQUENCES)) + .andExpect(jsonPath("\$[0].submissionId").value("custom0")) + .andExpect(jsonPath("\$[0].accession").value(accessions.first())) + .andExpect(jsonPath("\$[0].version").value(2)) + } + @ParameterizedTest(name = "GIVEN {0} THEN throws error \"{5}\"") @MethodSource("badRequestForRevision") fun `GIVEN invalid data THEN throws bad request`( @@ -231,7 +290,7 @@ class ReviseEndpointTest( SubmitFiles.sequenceFileWith(name = "notSequencesFile"), status().isBadRequest, "Bad Request", - "Required part 'sequenceFile' is not present.", + "Submissions for organism $DEFAULT_ORGANISM require a sequence file.", ), Arguments.of( "wrong extension for metadata file", diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt index cc97aa4fcc..eba76f6dd8 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt @@ -33,14 +33,16 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post class SubmissionControllerClient(private val mockMvc: MockMvc, private val objectMapper: ObjectMapper) { fun submit( metadataFile: MockMultipartFile, - sequencesFile: MockMultipartFile, + sequencesFile: MockMultipartFile? = null, organism: String = DEFAULT_ORGANISM, groupId: Int, dataUseTerm: DataUseTerms = DataUseTerms.Open, jwt: String? = jwtForDefaultUser, ): ResultActions = mockMvc.perform( multipart(addOrganismToPath("/submit", organism = organism)) - .file(sequencesFile) + .apply { + sequencesFile?.let { file(sequencesFile) } + } .file(metadataFile) .param("groupId", groupId.toString()) .param("dataUseTermsType", dataUseTerm.type.name) @@ -175,9 +177,11 @@ class SubmissionControllerClient(private val mockMvc: MockMvc, private val objec .content( """{ "accessionVersionsFilter": ${serialize(accessionVersionsFilter)}, - ${submitterNamesFilter?.let { - """"submitterNamesFilter": [${it.joinToString(",") { name -> "\"$name\"" }}],""" - } ?: ""} + ${ + submitterNamesFilter?.let { + """"submitterNamesFilter": [${it.joinToString(",") { name -> "\"$name\"" }}],""" + } ?: "" + } "scope": "$scope" }""", ) @@ -243,12 +247,14 @@ class SubmissionControllerClient(private val mockMvc: MockMvc, private val objec fun reviseSequenceEntries( metadataFile: MockMultipartFile, - sequencesFile: MockMultipartFile, + sequencesFile: MockMultipartFile?, organism: String = DEFAULT_ORGANISM, jwt: String? = jwtForDefaultUser, ): ResultActions = mockMvc.perform( multipart(addOrganismToPath("/revise", organism = organism)) - .file(sequencesFile) + .apply { + sequencesFile?.let { file(sequencesFile) } + } .file(metadataFile) .withAuth(jwt), ) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt index 1cc5ab2299..35aa6c2af0 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt @@ -11,6 +11,7 @@ import org.loculus.backend.api.EditedSequenceEntryData import org.loculus.backend.api.GeneticSequence import org.loculus.backend.api.GetSequenceResponse import org.loculus.backend.api.Organism +import org.loculus.backend.api.OriginalData import org.loculus.backend.api.ProcessedData import org.loculus.backend.api.ProcessingResult import org.loculus.backend.api.SequenceEntryStatus @@ -24,6 +25,7 @@ import org.loculus.backend.controller.DEFAULT_GROUP import org.loculus.backend.controller.DEFAULT_ORGANISM import org.loculus.backend.controller.DEFAULT_PIPELINE_VERSION import org.loculus.backend.controller.DEFAULT_USER_NAME +import org.loculus.backend.controller.ORGANISM_WITHOUT_SEQUENCES import org.loculus.backend.controller.OTHER_ORGANISM import org.loculus.backend.controller.expectNdjsonAndGetContent import org.loculus.backend.controller.generateJwtFor @@ -57,14 +59,19 @@ class SubmissionConvenienceClient( .createNewGroup(group = DEFAULT_GROUP, jwt = generateJwtFor(username)) .andGetGroupId() - val isMultiSegmented = backendConfig - .getInstanceConfig(Organism(organism)) + val instanceConfig = backendConfig.getInstanceConfig(Organism(organism)) + + val isMultiSegmented = instanceConfig .referenceGenomes .nucleotideSequences.size > 1 + val doesNotAllowSequenceFile = !instanceConfig.schema.allowSubmissionOfConsensusSequences + val submit = client.submit( DefaultFiles.metadataFile, - if (isMultiSegmented) { + if (doesNotAllowSequenceFile) { + null + } else if (isMultiSegmented) { DefaultFiles.sequencesFileMultiSegmented } else { DefaultFiles.sequencesFile @@ -153,6 +160,9 @@ class SubmissionConvenienceClient( accession = it.accession, ) + ORGANISM_WITHOUT_SEQUENCES -> PreparedProcessedData.successfullyProcessed(accession = it.accession) + .copy(data = defaultProcessedDataWithoutSequences) + else -> throw Exception("Test issue: There is no mapping of processed data for organism $organism") } }.toTypedArray(), @@ -303,15 +313,33 @@ class SubmissionConvenienceClient( ), ).processingResultCounts[processingResult]!!.toInt() - fun submitDefaultEditedData(accessions: List, userName: String = DEFAULT_USER_NAME) { + fun submitEditedData( + accessions: List, + organism: String = DEFAULT_ORGANISM, + userName: String = DEFAULT_USER_NAME, + editedData: OriginalData, + ) { accessions.forEach { accession -> client.submitEditedSequenceEntryVersion( - EditedSequenceEntryData(accession, 1L, defaultOriginalData), + EditedSequenceEntryData(accession, 1L, editedData), jwt = generateJwtFor(userName), + organism = organism, ) + .andExpect(status().isNoContent) } } + fun submitDefaultEditedData( + accessions: List, + organism: String = DEFAULT_ORGANISM, + userName: String = DEFAULT_USER_NAME, + ) = submitEditedData( + accessions, + organism = organism, + userName = userName, + editedData = defaultOriginalData, + ) + fun approveProcessedSequenceEntries( accessionVersionsFilter: List, organism: String = DEFAULT_ORGANISM, diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionJourneyTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionJourneyTest.kt index aa501cf1f8..1a2b006b8f 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionJourneyTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionJourneyTest.kt @@ -1,6 +1,7 @@ package org.loculus.backend.controller.submission import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.anEmptyMap import org.hamcrest.Matchers.containsInAnyOrder import org.hamcrest.Matchers.empty import org.hamcrest.Matchers.`is` @@ -8,6 +9,7 @@ import org.junit.jupiter.api.Test import org.loculus.backend.api.AccessionVersion import org.loculus.backend.api.AccessionVersionInterface import org.loculus.backend.api.GeneticSequence +import org.loculus.backend.api.OriginalData import org.loculus.backend.api.ProcessedData import org.loculus.backend.api.Status.APPROVED_FOR_RELEASE import org.loculus.backend.api.Status.IN_PROCESSING @@ -15,6 +17,7 @@ import org.loculus.backend.api.Status.PROCESSED import org.loculus.backend.api.Status.RECEIVED import org.loculus.backend.controller.DEFAULT_ORGANISM import org.loculus.backend.controller.EndpointTest +import org.loculus.backend.controller.ORGANISM_WITHOUT_SEQUENCES import org.loculus.backend.controller.OTHER_ORGANISM import org.loculus.backend.controller.assertHasError import org.loculus.backend.controller.assertStatusIs @@ -170,6 +173,75 @@ class SubmissionJourneyTest(@Autowired val convenienceClient: SubmissionConvenie ) } + @Test + fun `Entries without sequences from submission, over edit and approval ending in status 'APPROVED_FOR_RELEASE'`() { + val accessions = convenienceClient.submitDefaultFiles(organism = ORGANISM_WITHOUT_SEQUENCES) + .submissionIdMappings + .map { it.accession } + + val getSequenceEntry = { + convenienceClient.getSequenceEntry( + accession = accessions.first(), + version = 1, + organism = ORGANISM_WITHOUT_SEQUENCES, + ) + } + + getSequenceEntry().assertStatusIs(RECEIVED) + + convenienceClient.extractUnprocessedData(organism = ORGANISM_WITHOUT_SEQUENCES) + convenienceClient.submitProcessedData( + accessions.map { + PreparedProcessedData.withErrors(accession = it) + .copy(data = defaultProcessedDataWithoutSequences) + }, + organism = ORGANISM_WITHOUT_SEQUENCES, + ) + + getSequenceEntry().assertStatusIs(PROCESSED) + .assertHasError(true) + + convenienceClient.submitEditedData( + accessions, + organism = ORGANISM_WITHOUT_SEQUENCES, + editedData = OriginalData( + metadata = defaultOriginalData.metadata, + unalignedNucleotideSequences = emptyMap(), + ), + ) + getSequenceEntry().assertStatusIs(RECEIVED) + + convenienceClient.extractUnprocessedData(organism = ORGANISM_WITHOUT_SEQUENCES) + getSequenceEntry().assertStatusIs(IN_PROCESSING) + + convenienceClient.submitProcessedData( + accessions.map { + PreparedProcessedData.successfullyProcessed(accession = it) + .copy(data = defaultProcessedDataWithoutSequences) + }, + organism = ORGANISM_WITHOUT_SEQUENCES, + ) + getSequenceEntry().assertStatusIs(PROCESSED) + .assertHasError(false) + + convenienceClient.approveProcessedSequenceEntries( + accessions.map { + AccessionVersion(it, 1) + }, + organism = ORGANISM_WITHOUT_SEQUENCES, + ) + getSequenceEntry().assertStatusIs(APPROVED_FOR_RELEASE) + + val releasedData = convenienceClient.getReleasedData(organism = ORGANISM_WITHOUT_SEQUENCES) + assertThat(releasedData.size, `is`(DefaultFiles.NUMBER_OF_SEQUENCES)) + val releasedDatum = releasedData.first() + assertThat(releasedDatum.unalignedNucleotideSequences, `is`(anEmptyMap())) + assertThat(releasedDatum.alignedNucleotideSequences, `is`(anEmptyMap())) + assertThat(releasedDatum.alignedAminoAcidSequences, `is`(anEmptyMap())) + assertThat(releasedDatum.nucleotideInsertions, `is`(anEmptyMap())) + assertThat(releasedDatum.aminoAcidInsertions, `is`(anEmptyMap())) + } + private fun getAccessionVersionsOfProcessedData(processedData: List>) = processedData .map { it.metadata } .map { it["accessionVersion"]!!.asText() } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt index ecc5b55c2c..7f5fea134c 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt @@ -17,6 +17,7 @@ import org.loculus.backend.api.Organism import org.loculus.backend.config.BackendConfig import org.loculus.backend.controller.DEFAULT_ORGANISM import org.loculus.backend.controller.EndpointTest +import org.loculus.backend.controller.ORGANISM_WITHOUT_SEQUENCES import org.loculus.backend.controller.OTHER_ORGANISM import org.loculus.backend.controller.expectUnauthorizedResponse import org.loculus.backend.controller.generateJwtFor @@ -195,6 +196,54 @@ class SubmitEndpointTest( .andExpect(jsonPath("\$.detail", containsString(expectedMessage))) } + @Test + fun `GIVEN no sequence file for organism that requires one THEN returns bad request`() { + submissionControllerClient.submit( + metadataFile = DefaultFiles.metadataFile, + sequencesFile = null, + organism = DEFAULT_ORGANISM, + groupId = groupId, + ) + .andExpect(status().isBadRequest) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) + .andExpect( + jsonPath("\$.detail").value("Submissions for organism $DEFAULT_ORGANISM require a sequence file."), + ) + } + + @Test + fun `GIVEN sequence file for organism without sequences THEN returns bad request`() { + submissionControllerClient.submit( + metadataFile = DefaultFiles.metadataFile, + sequencesFile = DefaultFiles.sequencesFile, + organism = ORGANISM_WITHOUT_SEQUENCES, + groupId = groupId, + ) + .andExpect(status().isBadRequest) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) + .andExpect( + jsonPath( + "\$.detail", + ).value("Sequence uploads are not allowed for organism $ORGANISM_WITHOUT_SEQUENCES."), + ) + } + + @Test + fun `GIVEN no sequence file for organism without sequences THEN data is accepted`() { + submissionControllerClient.submit( + metadataFile = DefaultFiles.metadataFile, + sequencesFile = null, + organism = ORGANISM_WITHOUT_SEQUENCES, + groupId = groupId, + ) + .andExpect(status().isOk) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) + .andExpect(jsonPath("\$.length()").value(NUMBER_OF_SEQUENCES)) + .andExpect(jsonPath("\$[0].submissionId").value("custom0")) + .andExpect(jsonPath("\$[0].accession", containsString(backendConfig.accessionPrefix))) + .andExpect(jsonPath("\$[0].version").value(1)) + } + companion object { @JvmStatic @@ -244,7 +293,7 @@ class SubmitEndpointTest( SubmitFiles.sequenceFileWith(name = "notSequencesFile"), status().isBadRequest, "Bad Request", - "Required part 'sequenceFile' is not present.", + "Submissions for organism $DEFAULT_ORGANISM require a sequence file.", DEFAULT_ORGANISM, DataUseTerms.Open, ), diff --git a/backend/src/test/resources/backend_config.json b/backend/src/test/resources/backend_config.json index 375a59bb4e..79e806c640 100644 --- a/backend/src/test/resources/backend_config.json +++ b/backend/src/test/resources/backend_config.json @@ -22,6 +22,7 @@ }, "schema": { "organismName": "Test", + "allowSubmissionOfConsensusSequences": true, "metadata": [ { "name": "date", @@ -124,6 +125,7 @@ }, "schema": { "organismName": "Test", + "allowSubmissionOfConsensusSequences": true, "metadata": [ { "name": "date", @@ -181,6 +183,45 @@ } ] } + }, + "dummyOrganismWithoutSequences": { + "referenceGenomes": { + "nucleotideSequences": [], + "genes": [] + }, + "schema": { + "organismName": "Test without sequences", + "allowSubmissionOfConsensusSequences": false, + "metadata": [ + { + "name": "date", + "type": "date", + "required": true + }, + { + "name": "region", + "type": "string", + "autocomplete": true, + "required": true + }, + { + "name": "country", + "type": "string", + "autocomplete": true, + "required": true + }, + { + "name": "division", + "type": "string", + "autocomplete": true + }, + { + "name": "host", + "type": "string", + "autocomplete": true + } + ] + } } } } diff --git a/backend/src/test/resources/backend_config_single_segment.json b/backend/src/test/resources/backend_config_single_segment.json index d084daa578..d9212eddc4 100644 --- a/backend/src/test/resources/backend_config_single_segment.json +++ b/backend/src/test/resources/backend_config_single_segment.json @@ -76,4 +76,4 @@ } } } -} \ No newline at end of file +}