Merge pull request #359 from galaxyproject/add_refdata_from_s3

Added support for loading refdata from s3
galaxyproject · Aug 9, 2022 · 4eb16d5 · 4eb16d5
2 parents 448176b + 60ae145
commit 4eb16d5
Show file tree

Hide file tree

Showing 12 changed files with 180 additions and 174 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -47,7 +47,7 @@ jobs:
       - name: Helm repo add galaxy
         run: helm repo add galaxy https://github.com/CloudVE/helm-charts/raw/master
       - name: Helm install Galaxy
-        run: time bash -c 'helm install --create-namespace -n galaxy galaxy ./galaxy --set persistence.accessMode="ReadWriteOnce" --set persistence.size="5Gi" --set postgresql.deploy=true --set resources.requests.memory=0Mi,resources.requests.cpu=0m --wait --timeout=600s'
+        run: time bash -c 'helm install --create-namespace -n galaxy galaxy ./galaxy --set persistence.accessMode="ReadWriteOnce" --set resources.requests.memory=0Mi,resources.requests.cpu=0m --wait --timeout=600s'
       - name: Get events
         run: kubectl get events -n galaxy; kubectl get events -n csi-drivers;
         if: always()

diff --git a/README.md b/README.md
diff --git a/galaxy/Chart.yaml b/galaxy/Chart.yaml
@@ -20,3 +20,10 @@ dependencies:
     alias: cvmfs
     tags:
       - deploy-cvmfs
+  - name: csi-s3
+    repository: https://raw.githubusercontent.com/cloudve/helm-charts/master/
+    version: 0.31.3
+    condition: s3csi.deploy
+    alias: s3csi
+    tags:
+      - deploy-s3csi
diff --git a/galaxy/templates/_helpers.tpl b/galaxy/templates/_helpers.tpl
@@ -161,7 +161,7 @@ Creates the bash command for the handlers to wait for init scripts
 until [ -f /galaxy/server/config/mutable/db_init_done_{{$.Release.Revision}} ]; do echo "waiting for DB initialization"; sleep 1; done;
 until [ -f /galaxy/server/config/mutable/init_mounts_done_{{$.Release.Revision}} ]; do echo "waiting for copying onto NFS"; sleep 1; done;
 {{ if .Values.setupJob.downloadToolConfs.enabled }}
-until [ -f /galaxy/server/config/mutable/init_clone_done_{{$.Release.Revision}} ]; do echo "waiting for CVMFS copying"; sleep 1; done;
+until [ -f /galaxy/server/config/mutable/init_clone_done_{{$.Release.Revision}} ]; do echo "waiting for refdata copying"; sleep 1; done;
 {{- end }}
 {{- end -}}
 

diff --git a/galaxy/templates/deployment-job.yaml b/galaxy/templates/deployment-job.yaml
@@ -142,9 +142,9 @@ spec:
               mountPath: {{ $.Values.setupJob.downloadToolConfs.volume.mountPath }}
               subPath: {{ $.Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
-            {{- if $.Values.cvmfs.enabled }}
-            {{- range $key, $entry := $.Values.cvmfs.galaxyPersistentVolumeClaims }}
-            - name: cvmfs-gxy-{{ $key }}
+            {{- if $.Values.refdata.enabled }}
+            {{- range $key, $entry := $.Values.refdata.galaxyPersistentVolumeClaims }}
+            - name: refdata-gxy-{{ $key }}
               mountPath: {{ $entry.mountPath }}
             {{- end -}}
             {{- end }}
@@ -187,11 +187,11 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        {{- if $.Values.cvmfs.enabled }}
-        {{- range $key, $entry := $.Values.cvmfs.galaxyPersistentVolumeClaims }}
-        - name: cvmfs-gxy-{{ $key }}
+        {{- if $.Values.refdata.enabled }}
+        {{- range $key, $entry := $.Values.refdata.galaxyPersistentVolumeClaims }}
+        - name: refdata-gxy-{{ $key }}
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
         {{- end }}
         {{- end }}
         {{- if $.Values.extraVolumes }}

diff --git a/galaxy/templates/deployment-web.yaml b/galaxy/templates/deployment-web.yaml
@@ -133,9 +133,9 @@ spec:
               mountPath: {{ .Values.setupJob.downloadToolConfs.volume.mountPath }}
               subPath: {{ .Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
-            {{- if .Values.cvmfs.enabled }}
-            {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-            - name: cvmfs-gxy-{{ $key }}
+            {{- if .Values.refdata.enabled }}
+            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+            - name: refdata-gxy-{{ $key }}
               mountPath: {{ $entry.mountPath }}
             {{- end -}}
             {{- end }}
@@ -181,11 +181,11 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        {{- if .Values.cvmfs.enabled }}
-        {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-        - name: cvmfs-gxy-{{ $key }}
+        {{- if .Values.refdata.enabled }}
+        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+        - name: refdata-gxy-{{ $key }}
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
         {{- end }}
         {{- end }}
         {{- if .Values.extraVolumes }}

diff --git a/galaxy/templates/deployment-workflow.yaml b/galaxy/templates/deployment-workflow.yaml
@@ -139,9 +139,9 @@ spec:
               mountPath: {{ .Values.setupJob.downloadToolConfs.volume.mountPath }}
               subPath: {{ .Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
-            {{- if .Values.cvmfs.enabled }}
-            {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-            - name: cvmfs-gxy-{{ $key }}
+            {{- if .Values.refdata.enabled }}
+            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+            - name: refdata-gxy-{{ $key }}
               mountPath: {{ $entry.mountPath }}
             {{- end -}}
             {{- end }}
@@ -184,11 +184,11 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        {{- if .Values.cvmfs.enabled }}
-        {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-        - name: cvmfs-gxy-{{ $key }}
+        {{- if .Values.refdata.enabled }}
+        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+        - name: refdata-gxy-{{ $key }}
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
         {{- end }}
         {{- end }}
         {{- if .Values.extraVolumes }}

diff --git a/galaxy/templates/jobs-init.yaml b/galaxy/templates/jobs-init.yaml
@@ -75,9 +75,9 @@ spec:
             - name: galaxy-data
               mountPath: /galaxy/server/config/mutable/
               subPath: config
-            {{- if .Values.cvmfs.enabled }}
-            {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-            - name: cvmfs-gxy-{{ $key }}
+            {{- if .Values.refdata.enabled }}
+            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+            - name: refdata-gxy-{{ $key }}
               mountPath: {{ $entry.mountPath }}
             {{- end -}}
             {{- end }}
@@ -115,11 +115,11 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        {{- if .Values.cvmfs.enabled }}
-        {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-        - name: cvmfs-gxy-{{ $key }}
+        {{- if .Values.refdata.enabled }}
+        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+        - name: refdata-gxy-{{ $key }}
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
         {{- end }}
         {{- end }}
         {{- if .Values.extraVolumes }}
@@ -182,9 +182,9 @@ spec:
             - name: galaxy-data
               mountPath: /galaxy/server/config/mutable/
               subPath: config
-            {{- if .Values.cvmfs.enabled }}
-            {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-            - name: cvmfs-gxy-{{ $key }}
+            {{- if .Values.refdata.enabled }}
+            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+            - name: refdata-gxy-{{ $key }}
               mountPath: {{ $entry.mountPath }}
             {{- end -}}
             {{- end }}
@@ -248,11 +248,11 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        {{- if .Values.cvmfs.enabled }}
-        {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-        - name: cvmfs-gxy-{{ $key }}
+        {{- if .Values.refdata.enabled }}
+        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+        - name: refdata-gxy-{{ $key }}
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
         {{- end }}
         {{- end }}
         {{- if .Values.extraVolumes }}

diff --git a/galaxy/templates/pv-s3fs.yaml b/galaxy/templates/pv-s3fs.yaml
@@ -0,0 +1,33 @@
+{{- if and .Values.refdata.enabled (eq .Values.refdata.type "s3csi") }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ include "galaxy.fullname" $ }}-refdata-gxy-pv
+  labels:
+    {{- include "galaxy.labels" . | nindent 4 }}
+spec:
+  accessModes:
+  - ReadOnlyMany
+  capacity:
+    storage: 30Gi
+  csi:
+    controllerPublishSecretRef:
+      name: csi-s3-secret
+      namespace: {{ .Release.Namespace }}
+    driver: ru.yandex.s3.csi
+    nodePublishSecretRef:
+      name: csi-s3-secret
+      namespace: {{ .Release.Namespace }}
+    nodeStageSecretRef:
+      name: csi-s3-secret
+      namespace: {{ .Release.Namespace }}
+    volumeAttributes:
+      bucket: {{ .Values.s3csi.storageClass.singleBucket }}
+      capacity: "32212254720"
+      mounter: {{ .Values.s3csi.storageClass.mounter }}
+      options: {{ .Values.s3csi.storageClass.mountOptions }}
+    volumeHandle: {{ .Values.s3csi.storageClass.singleBucket }}{{ .Values.s3csi.secret.prefix }}
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: {{ .Values.s3csi.storageClass.name }}
+  volumeMode: Filesystem
+{{- end }}
diff --git a/galaxy/templates/pvcs-cvmfs.yaml b/galaxy/templates/pvcs-cvmfs.yaml
@@ -1,9 +1,9 @@
-{{- if .Values.cvmfs.enabled }}
-{{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims -}}
+{{- if .Values.refdata.enabled }}
+{{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims -}}
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: {{ include "galaxy.fullname" $ }}-cvmfs-gxy-{{ $key }}-pvc
+  name: {{ include "galaxy.fullname" $ }}-refdata-gxy-{{ $key }}-pvc
   labels:
     {{- include "galaxy.labels" $ | nindent 4 }}
 spec:
@@ -12,6 +12,9 @@ spec:
   resources:
     requests:
       storage: {{ $entry.storage }}
+{{- if eq $.Values.refdata.type "s3csi" }}
+  volumeName: {{ include "galaxy.fullname" $ }}-refdata-gxy-pv
+{{- end }}
   storageClassName: {{ $entry.storageClassName }}
 ---
 {{- end }}

diff --git a/galaxy/values.yaml b/galaxy/values.yaml
@@ -118,7 +118,7 @@ persistence:
   storageClass: ""
   existingClaim: null
   accessMode: ReadWriteMany
-  size: 10Gi
+  size: 5Gi
   mountPath: /galaxy/server/database
 
 extraVolumes: []
@@ -246,21 +246,24 @@ postgresql:
     #    matchLabels:
     #      label-key: label-value
 
-cvmfs:
-  repositories:
-    cvmfs-gxy-data: data.galaxyproject.org
-    # cvmfs-gxy-cloud: cloud.galaxyproject.org
-  enabled: false
-  deploy: false
+refdata:
+  enabled: true
+  type: s3csi
   galaxyPersistentVolumeClaims:
     data:
-      storage: 1Gi
-      storageClassName: cvmfs-gxy-data
+      storage: 10Gi
+      storageClassName: refdata-gxy-data
       mountPath: /cvmfs/data.galaxyproject.org
     # cloud:
     #   storage: 1Gi
     #   storageClassName: cvmfs-gxy-cloud
     #   mountPath: /cvmfs/cloud.galaxyproject.org
+
+cvmfs:
+  repositories:
+    cvmfs-gxy-data: data.galaxyproject.org
+    # cvmfs-gxy-cloud: cloud.galaxyproject.org
+  deploy: false
   cache:
     alienCache:
       enabled: true
@@ -270,6 +273,25 @@ cvmfs:
     preload:
       enabled: false
 
+s3csi:
+  deploy: true
+  images:
+    csi: cloudve/csi-s3:0.31.3
+  storageClass:
+    name: refdata-gxy-data
+    mounter: s3fs
+    singleBucket: biorefdata
+    mountOptions: "-o use_cache=/tmp -o endpoint=ap-southeast-2 -o public_bucket=1 -o enable_noobj_cache -o no_check_certificate -o kernel_cache -o ensure_diskfree=5000"
+    # additional optimizations for s3fs
+    # -o max_background=1000 -o max_stat_cache_size=1000000 -o multipart_size=52 -o parallel_count=30 -o multireq_max=30 -o dbglevel=warn
+    # future settings for geesefs
+    # mountOptions: "--memory-limit 4000 --dir-mode 0777 --file-mode 0666 --cache /tmp/geesecache --debug --debug_fuse --stat-cache-ttl 9m0s --cache-to-disk-hits 1"
+    # --no-dir-object --no-implicit-dir --stat-cache-ttl 120m0s --max-disk-cache-fd 4096
+  secret:
+    endpoint: https://s3.ap-southeast-2.amazonaws.com
+    usePrefix: true
+    prefix: /galaxy/v1/data.galaxyproject.org
+
 # When this flag is set to true, all configs will be set in secrets,
 # when it is set to false, all configs will be set in configmaps
 useSecretConfigs: false
@@ -286,9 +308,9 @@ configs:
         k8s_use_service_account: true
         k8s_persistent_volume_claims: |-
           {{ template "galaxy.pvcname" . -}}:{{ .Values.persistence.mountPath -}}
-          {{- if .Values.cvmfs.enabled -}}
-          {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims -}}
-          ,{{- template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc:{{ $entry.mountPath -}}
+          {{- if .Values.refdata.enabled -}}
+          {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims -}}
+          ,{{- template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc:{{ $entry.mountPath -}}
           {{- end -}}
           {{- end -}}
           {{- if .Values.setupJob.downloadToolConfs.enabled -}}
@@ -379,19 +401,19 @@ configs:
       tool_data_table_config_path: |-
         {{ if .Values.setupJob.downloadToolConfs.enabled }}
         {{- .Values.setupJob.downloadToolConfs.volume.mountPath }}/config/shed_tool_data_table_conf.xml
-        {{- if .Values.cvmfs.enabled -}}
-        ,{{- .Values.cvmfs.galaxyPersistentVolumeClaims.data.mountPath}}/managed/location/tool_data_table_conf.xml,{{.Values.cvmfs.galaxyPersistentVolumeClaims.data.mountPath}}/byhand/location/tool_data_table_conf.xml
+        {{- if .Values.refdata.enabled -}}
+        ,{{- .Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/managed/location/tool_data_table_conf.xml,{{.Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/byhand/location/tool_data_table_conf.xml
         {{- end }}
         {{- else }}
-        {{- if .Values.cvmfs.enabled -}}
-        {{- .Values.cvmfs.galaxyPersistentVolumeClaims.data.mountPath}}/managed/location/tool_data_table_conf.xml,{{.Values.cvmfs.galaxyPersistentVolumeClaims.data.mountPath}}/byhand/location/tool_data_table_conf.xml
+        {{- if .Values.refdata.enabled -}}
+        {{- .Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/managed/location/tool_data_table_conf.xml,{{.Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/byhand/location/tool_data_table_conf.xml
         {{- end }}
         {{- end }}
       tool_dependency_dir: "{{.Values.persistence.mountPath}}/deps"
       job_config_file: "/galaxy/server/config/job_conf.yml"
       builds_file_path: |-
-        {{ if .Values.cvmfs.enabled }}
-        {{ .Values.cvmfs.galaxyPersistentVolumeClaims.data.mountPath}}/managed/location/builds.txt
+        {{ if .Values.refdata.enabled }}
+        {{- .Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/managed/location/builds.txt
         {{- end }}
       containers_resolvers_config_file: "/galaxy/server/config/container_resolvers_conf.xml"
       workflow_schedulers_config_file: "/galaxy/server/config/workflow_schedulers_conf.xml"
@@ -405,8 +427,8 @@ configs:
       nginx_x_accel_redirect_base: >-
         {{include "galaxy.add_trailing_slash" .Values.ingress.path}}_x_accel_redirect
       len_file_path: |-
-        {{ if .Values.cvmfs.enabled -}}
-        {{ .Values.cvmfs.galaxyPersistentVolumeClaims.data.mountPath}}/managed/len/ucsc
+        {{ if .Values.refdata.enabled -}}
+        {{- .Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/managed/len/ucsc
         {{- end }}
   container_resolvers_conf.xml: |
     <containers_resolvers>
@@ -443,7 +465,7 @@ configs:
     toolshed.g2.bx.psu.edu/repos/iuc/snpeff/snpEff/4.3+T.galaxy1
   build_sites.yml:
     - type: ucsc
-      file: "{{if .Values.cvmfs.enabled}}{{.Values.cvmfs.galaxyPersistentVolumeClaims.data.mountPath -}}/managed/location/ucsc_build_sites.txt{{else}}/galaxy/server/tool-data/shared/ucsc/ucsc_build_sites.txt.sample{{end}}"
+      file: "{{if .Values.refdata.enabled}}{{.Values.refdata.galaxyPersistentVolumeClaims.data.mountPath -}}/managed/location/ucsc_build_sites.txt{{else}}/galaxy/server/tool-data/shared/ucsc/ucsc_build_sites.txt.sample{{end}}"
       display: [main,archaea,ucla]
     - type: gbrowse
       file: "/galaxy/server/tool-data/shared/gbrowse/gbrowse_build_sites.txt"

diff --git a/scripts/values_table.py b/scripts/values_table.py
@@ -29,7 +29,7 @@
     'extraInitContainers': 'extraInitContainers.[]',
     'extraEnv': 'extraInitContainers.[]',
     'jobs\\.rules\\..*': 'jobs.rules',
-    'cvmfs\\.galaxyPersistentVolumeClaims\\..*': 'cvmfs.galaxyPersistentVolumeClaims.{}',
+    'refdata\\.galaxyPersistentVolumeClaims\\..*': 'refdata.galaxyPersistentVolumeClaims.{}',
     'jobHandlers\\..*': 'jobHandlers.{}',
     'webHandlers\\..*': 'webHandlers.{}',
     'workflowHandlers\\..*': 'workflowHandlers.{}',
@@ -48,8 +48,10 @@
                 "useSecretConfigs",
                 "configs.",
                 "jobs.",
+                "refdata.deploy",
+                "refdata.enabled",
+                "refdata.",
                 "cvmfs.deploy",
-                "cvmfs.enabled",
                 "cvmfs.",
                 "setupJob.",
                 "ingress.",