chore(deps): SILO v0.5.0 (#3529)

* chore(deps): SILO v0.5.0 (#3529) * refactor: remove remains of pango_lineage from source code (#3529)
loculus-project · Jan 16, 2025 · 9aa28ab · 9aa28ab
1 parent f3d3b8b
commit 9aa28ab
Show file tree

Hide file tree

Showing 17 changed files with 16 additions and 70 deletions.
diff --git a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt
@@ -44,9 +44,6 @@ enum class MetadataType {
     @JsonProperty("date")
     DATE,
 
-    @JsonProperty("pango_lineage")
-    PANGO_LINEAGE,
-
     @JsonProperty("boolean")
     BOOLEAN,
 

diff --git a/...src/main/kotlin/org/loculus/backend/service/submission/ProcessedSequenceEntryValidator.kt b/...src/main/kotlin/org/loculus/backend/service/submission/ProcessedSequenceEntryValidator.kt
@@ -20,8 +20,6 @@ import java.time.format.DateTimeFormatter
 import java.time.format.DateTimeParseException
 
 private const val DATE_FORMAT = "yyyy-MM-dd"
-private const val PANGO_LINEAGE_REGEX_PATTERN = "[a-zA-Z]{1,3}(\\.\\d{1,3}){0,3}"
-private val pangoLineageRegex = Regex(PANGO_LINEAGE_REGEX_PATTERN)
 
 interface Symbol {
     val symbol: Char
@@ -100,9 +98,6 @@ private fun <T> validateNoUnknownInMetaData(data: Map<String, T>, known: List<St
     }
 }
 
-private fun isValidPangoLineage(pangoLineageCandidate: String): Boolean =
-    pangoLineageCandidate.matches(pangoLineageRegex)
-
 private fun validateKnownMetadataField(metadata: BaseMetadata, processedMetadataMap: MetadataMap): MetadataMap {
     val fieldName = metadata.name
     val fieldValue = processedMetadataMap[fieldName]
@@ -141,17 +136,6 @@ private fun validateType(fieldValue: JsonNode, metadata: BaseMetadata) {
             return
         }
 
-        MetadataType.PANGO_LINEAGE -> {
-            if (!isValidPangoLineage(fieldValue.asText())) {
-                throw ProcessingValidationException(
-                    "Expected type 'pango_lineage' for field '${metadata.name}', " +
-                        "found value '$fieldValue'. " +
-                        "A pango lineage must be of the form $PANGO_LINEAGE_REGEX_PATTERN, e.g. 'XBB' or 'BA.1.5'.",
-                )
-            }
-            return
-        }
-
         else -> {}
     }
 

diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/PreparedProcessedData.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/PreparedProcessedData.kt
@@ -212,15 +212,6 @@ object PreparedProcessedData {
         ),
     )
 
-    fun withWrongPangoLineageFormat(accession: Accession) = defaultSuccessfulSubmittedData.copy(
-        accession = accession,
-        data = defaultProcessedData.copy(
-            metadata = defaultProcessedData.metadata + mapOf(
-                "pangoLineage" to TextNode("A.5.invalid"),
-            ),
-        ),
-    )
-
     fun withWrongBooleanFormat(accession: Accession) = defaultSuccessfulSubmittedData.copy(
         accession = accession,
         data = defaultProcessedData.copy(

diff --git a/.../test/kotlin/org/loculus/backend/controller/submission/SubmitProcessedDataEndpointTest.kt b/.../test/kotlin/org/loculus/backend/controller/submission/SubmitProcessedDataEndpointTest.kt
@@ -506,15 +506,6 @@ class SubmitProcessedDataEndpointTest(
                 expectedErrorMessage =
                 "Expected type 'boolean' for field 'booleanColumn', found value '\"not a boolean\"'.",
             ),
-            InvalidDataScenario(
-                name = "data with wrong pango lineage format",
-                processedDataThatNeedsAValidAccession = PreparedProcessedData.withWrongPangoLineageFormat(
-                    accession = "DoesNotMatter",
-                ),
-                expectedErrorMessage =
-                "Expected type 'pango_lineage' for field 'pangoLineage', found value '\"A.5.invalid\"'. " +
-                    "A pango lineage must be of the form [a-zA-Z]{1,3}(\\.\\d{1,3}){0,3}, e.g. 'XBB' or 'BA.1.5'.",
-            ),
             InvalidDataScenario(
                 name = "data with explicit null for required field",
                 processedDataThatNeedsAValidAccession = PreparedProcessedData.withNullForFields(

diff --git a/backend/src/test/resources/backend_config.json b/backend/src/test/resources/backend_config.json
@@ -65,7 +65,7 @@
                     },
                     {
                         "name": "pangoLineage",
-                        "type": "pango_lineage",
+                        "type": "string",
                         "autocomplete": true
                     },
                     {
@@ -172,7 +172,7 @@
                     },
                     {
                         "name": "pangoLineage",
-                        "type": "pango_lineage",
+                        "type": "string",
                         "autocomplete": true
                     },
                     {

diff --git a/backend/src/test/resources/backend_config_single_segment.json b/backend/src/test/resources/backend_config_single_segment.json
@@ -65,7 +65,7 @@
                     },
                     {
                         "name": "pangoLineage",
-                        "type": "pango_lineage",
+                        "type": "string",
                         "autocomplete": true
                     },
                     {

diff --git a/docs/src/content/docs/for-administrators/setup-with-kubernetes.md b/docs/src/content/docs/for-administrators/setup-with-kubernetes.md
@@ -98,7 +98,7 @@ organisms:
                   type: string
                   autocomplete: true
                 - name: pangoLineage
-                  type: pango_lineage
+                  type: string
                   autocomplete: true
                   required: true
                 - name: insdcAccessionFull

diff --git a/kubernetes/loculus/silo_import_job.sh b/kubernetes/loculus/silo_import_job.sh
@@ -203,17 +203,17 @@ preprocessing() {
   cp "$new_input_data_path" "$silo_input_data_path"
 
   set +e
-  time /app/siloApi --preprocessing --preprocessingConfig=$preprocessing_config_file_merged
+  time /app/silo preprocessing --preprocessing-config=$preprocessing_config_file_merged
   exit_code=$?
   set -e
 
   if [ $exit_code -ne 0 ]; then
-    echo "SiloApi command failed with exit code $exit_code, cleaning up and exiting."
+    echo "silo command failed with exit code $exit_code, cleaning up and exiting."
     delete_all_input # Delete input so that we don't skip preprocessing next time due to hash equality
     exit $exit_code
   fi
 
-  echo "SiloApi command succeeded"
+  echo "silo command succeeded"
   echo "Removing touchfile $new_input_touchfile to indicate successful processing"
   rm "$new_input_touchfile"
 

diff --git a/kubernetes/loculus/templates/lapis-silo-database-config.yaml b/kubernetes/loculus/templates/lapis-silo-database-config.yaml
@@ -19,7 +19,6 @@ data:
 
   preprocessing_config.yaml: |
     ndjsonInputFilename: data.ndjson.zst
-    pangoLineageDefinitionFilename: pangolineage_alias.json
     referenceGenomeFilename: reference_genomes.json
 
   reference_genomes.json: |
@@ -32,7 +31,4 @@ data:
   silo_import_wrapper.sh: |
     {{ range $importScriptWrapperLines }}
     {{ . }}{{ end }}
-
-  pangolineage_alias.json: |
-    {{ $instance.pangolineage_alias | default dict | toJson }}
 {{- end }}
diff --git a/kubernetes/loculus/templates/lapis-silo-deployment.yaml b/kubernetes/loculus/templates/lapis-silo-deployment.yaml
@@ -33,7 +33,7 @@ spec:
           ports:
             - containerPort: 8081
           args:
-            - "--api"
+            - "api"
           volumeMounts:
             - name: lapis-silo-shared-data
               mountPath: /data
@@ -94,14 +94,11 @@ spec:
               mountPath: /preprocessing/input/reference_genomes.json
               subPath: reference_genomes.json
             - name: lapis-silo-database-config-processed
-              mountPath: /preprocessing/input/pangolineage_alias.json
-              subPath: pangolineage_alias.json
+              mountPath: /preprocessing/input/database_config.yaml
+              subPath: database_config.yaml
             - name: lapis-silo-database-config-processed
               mountPath: /app/preprocessing_config.yaml
               subPath: preprocessing_config.yaml
-            - name: lapis-silo-database-config-processed
-              mountPath: /app/database_config.yaml
-              subPath: database_config.yaml
             - name: lapis-silo-shared-data
               mountPath: /preprocessing/output
             - name: lapis-silo-input-data-cache

diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml
@@ -1527,7 +1527,7 @@ bannerMessage: "This is a demonstration environment. It may contain non-accurate
 welcomeMessageHTML: null
 additionalHeadHTML: ""
 images:
-  lapisSilo: "ghcr.io/genspectrum/lapis-silo:0.3.2"
+  lapisSilo: "ghcr.io/genspectrum/lapis-silo:0.5.0"
   lapis: "ghcr.io/genspectrum/lapis:0.3.10"
 secrets:
   smtp-password:

diff --git a/preprocessing/specification.md b/preprocessing/specification.md
@@ -123,7 +123,6 @@ The `metadata` field should contain a flat object consisting of the fields speci
 - `int` (integer)
 - `float`
 - `date` (supplied as a string with complete ISO-8601 date, e.g., "2023-08-30")
-- `pango_lineage` (supplied as a string with a properly formatted SARS-CoV-2 Pango lineage, e.g., "B.1.1.7")
 - `authors` (comma separated list of authors, treated as a string in the current prepro pipeline)
 
 #### Sequences

diff --git a/website/src/components/SearchPage/SearchForm.spec.tsx b/website/src/components/SearchPage/SearchForm.spec.tsx
@@ -26,7 +26,7 @@ const defaultSearchFormFilters: MetadataFilter[] = [
     },
     {
         name: 'field3',
-        type: 'pango_lineage',
+        type: 'string',
         label: 'Field 3',
         autocomplete: true,
         initiallyVisible: true,

diff --git a/website/src/components/SearchPage/SearchFullUI.spec.tsx b/website/src/components/SearchPage/SearchFullUI.spec.tsx
@@ -49,7 +49,7 @@ const defaultSearchFormFilters: MetadataFilter[] = [
     },
     {
         name: 'field3',
-        type: 'pango_lineage',
+        type: 'string',
         label: 'Field 3',
         autocomplete: true,
         initiallyVisible: true,

diff --git a/website/src/components/SearchPage/fields/FieldProps.tsx b/website/src/components/SearchPage/fields/FieldProps.tsx
@@ -10,5 +10,5 @@ export type FieldProps = {
     onFocus?: FocusEventHandler<HTMLInputElement | HTMLTextAreaElement>;
     onBlur?: FocusEventHandler<HTMLInputElement | HTMLTextAreaElement>;
     fieldValue: string;
-    type?: 'string' | 'boolean' | 'float' | 'int' | 'pango_lineage' | 'authors';
+    type?: 'string' | 'boolean' | 'float' | 'int' | 'authors';
 };
diff --git a/website/src/components/SearchPage/fields/NormalTextField.tsx b/website/src/components/SearchPage/fields/NormalTextField.tsx
@@ -10,7 +10,7 @@ export type NormalFieldProps = {
     onFocus?: FocusEventHandler<HTMLInputElement | HTMLTextAreaElement>;
     onBlur?: FocusEventHandler<HTMLInputElement | HTMLTextAreaElement>;
     fieldValue: string | number;
-    type?: 'string' | 'boolean' | 'float' | 'int' | 'pango_lineage' | 'authors';
+    type?: 'string' | 'boolean' | 'float' | 'int' | 'authors';
 };
 
 export const NormalTextField = forwardRef<HTMLInputElement, NormalFieldProps>((props, ref) => {

diff --git a/website/src/types/config.ts b/website/src/types/config.ts
@@ -4,16 +4,7 @@ import { mutationProportionCount, orderByType } from './lapis.ts';
 import { referenceGenomes } from './referencesGenomes.ts';
 
 // These metadata types need to be kept in sync with the backend config class `MetadataType` in Config.kt
-const metadataPossibleTypes = z.enum([
-    'string',
-    'date',
-    'int',
-    'float',
-    'pango_lineage',
-    'timestamp',
-    'boolean',
-    'authors',
-] as const);
+const metadataPossibleTypes = z.enum(['string', 'date', 'int', 'float', 'timestamp', 'boolean', 'authors'] as const);
 
 export const segmentedMutations = z.object({
     segment: z.string(),