diff --git a/.secrets.baseline b/.secrets.baseline index 8516eea5..eb6b0cc7 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2026-04-10T16:09:14Z", + "generated_at": "2026-04-23T10:54:31Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -870,7 +870,7 @@ "hashed_secret": "8e9e0a058630388dcbb497903ec1a5b03e3781f9", "is_secret": false, "is_verified": false, - "line_number": 53, + "line_number": 56, "type": "Secret Keyword", "verified_result": null }, @@ -878,7 +878,7 @@ "hashed_secret": "055abb8b464ae0ab2bb7e9d2fa150c641fee8334", "is_secret": false, "is_verified": false, - "line_number": 58, + "line_number": 61, "type": "Secret Keyword", "verified_result": null } @@ -1041,7 +1041,7 @@ "hashed_secret": "66cf0e57ff0c4392b01bfadcd28759c621938a02", "is_secret": false, "is_verified": false, - "line_number": 358, + "line_number": 360, "type": "Secret Keyword", "verified_result": null }, @@ -1049,7 +1049,7 @@ "hashed_secret": "64955456d46405ff355b3abc0cf74a4ff9bdf283", "is_secret": false, "is_verified": false, - "line_number": 361, + "line_number": 363, "type": "Secret Keyword", "verified_result": null }, @@ -1057,7 +1057,7 @@ "hashed_secret": "90c3cc8d4eae70f6a1e7c2a9c88a57d5d9623deb", "is_secret": false, "is_verified": false, - "line_number": 369, + "line_number": 371, "type": "Secret Keyword", "verified_result": null }, @@ -1065,7 +1065,7 @@ "hashed_secret": "8e9e0a058630388dcbb497903ec1a5b03e3781f9", "is_secret": false, "is_verified": false, - "line_number": 373, + "line_number": 375, "type": "Secret Keyword", "verified_result": null }, @@ -1073,7 +1073,7 @@ "hashed_secret": "055abb8b464ae0ab2bb7e9d2fa150c641fee8334", "is_secret": false, "is_verified": false, - "line_number": 377, + "line_number": 379, "type": "Secret Keyword", "verified_result": null } diff --git a/deployment-scripts/create_buckets.py b/deployment-scripts/create_buckets.py index 33ddefc9..3330b745 100644 --- a/deployment-scripts/create_buckets.py +++ b/deployment-scripts/create_buckets.py @@ -54,7 +54,8 @@ def get_s3_client(): "geoserver", "temp-upload", "inference-auxdata", - "generic-python-processor" + "generic-python-processor", + "pipeline-data" ] diff --git a/deployment-scripts/template/env.template.sh b/deployment-scripts/template/env.template.sh index 9b0be374..f609e6ca 100644 --- a/deployment-scripts/template/env.template.sh +++ b/deployment-scripts/template/env.template.sh @@ -26,6 +26,10 @@ export NON_COS_STORAGE_CLASS= export PVC_ACCESS_MODE=ReadWriteOnce export PIPELINES_V2_INFERENCE_ROOT_FOLDER_VALUE= +# Terrakit Cache Configuration +export TERRAKIT_CACHE_ENABLED_VALUE=true +export TERRAKIT_CACHE_DIR_VALUE=/pipeline/data/terrakit_cache + # Populate buckets with data export LULC_TILE_ROOT= export LULC_TILE_SHAPEFILE= @@ -68,6 +72,7 @@ export BUCKET_GEOSERVER=${DEPLOYMENT_ENV}-geoserver export BUCKET_TEMP_UPLOAD=${DEPLOYMENT_ENV}-temp-upload export BUCKET_INFERENCE_AUXDATA=${DEPLOYMENT_ENV}-inference-auxdata export BUCKET_GENERIC_PYTHON_PROCESSOR=${DEPLOYMENT_ENV}-generic-python-processor +export BUCKET_PIPELINE_DATA=${DEPLOYMENT_ENV}-pipeline-data # GPU_CONFIG diff --git a/deployment-scripts/values-file-generate.sh b/deployment-scripts/values-file-generate.sh index 2dab1c0a..9e0ea160 100755 --- a/deployment-scripts/values-file-generate.sh +++ b/deployment-scripts/values-file-generate.sh @@ -39,6 +39,7 @@ do -e "s|BUCKET_GFM_MLFLOW|${BUCKET_GFM_MLFLOW}|" \ -e "s|BUCKET_GEOSERVER|${BUCKET_GEOSERVER}|" \ -e "s|BUCKET_GENERIC_PYTHON_PROCESSOR|${BUCKET_GENERIC_PYTHON_PROCESSOR}|" \ + -e "s|BUCKET_PIPELINE_DATA|${BUCKET_PIPELINE_DATA}|" \ -e "s|COS_STORAGE_CLASS|${COS_STORAGE_CLASS}|" \ -e "s|SHARE_PIPELINE_PVC|${SHARE_PIPELINE_PVC}|" \ -e "s|OAUTH_TYPE|${OAUTH_TYPE}|" \ @@ -62,5 +63,7 @@ do -e "s|GEOSERVER_CM_PROXYBASEURL|${GEOSERVER_CM_PROXYBASEURL}|" \ -e "s|geospatial-studio|${CONTAINER_IMAGE_REPOSITORY}|" \ -e "s|PIPELINES_V2_INFERENCE_ROOT_FOLDER_VALUE|${PIPELINES_V2_INFERENCE_ROOT_FOLDER_VALUE}|" \ + -e "s|TERRAKIT_CACHE_ENABLED_VALUE|${TERRAKIT_CACHE_ENABLED_VALUE}|" \ + -e "s|TERRAKIT_CACHE_DIR_VALUE|${TERRAKIT_CACHE_DIR_VALUE}|" \ ${HELM_CHART_NAME}/values.yaml > workspace/$DEPLOYMENT_ENV/values/${HELM_CHART_NAME}/values.yaml done diff --git a/geospatial-studio/charts/geospatial-studio-pipelines/templates/secrets.yaml b/geospatial-studio/charts/geospatial-studio-pipelines/templates/secrets.yaml index b809e22d..d55c6599 100644 --- a/geospatial-studio/charts/geospatial-studio-pipelines/templates/secrets.yaml +++ b/geospatial-studio/charts/geospatial-studio-pipelines/templates/secrets.yaml @@ -21,6 +21,9 @@ data: gfmaas_api_key: {{ .Values.gateway.api_key | b64enc }} geoserver_username: {{ .Values.geoserver.username | b64enc }} geoserver_password: {{ .Values.geoserver.password | b64enc }} + {{- if .Values.redis.enabled }} + REDIS_URL: {{ include "geospatial-studio.redis.url" . | b64enc | quote }} + {{- end }} --- {{ if hasKey .Values "nasaEarthBearerToken"}} diff --git a/geospatial-studio/charts/geospatial-studio-pipelines/values.yaml b/geospatial-studio/charts/geospatial-studio-pipelines/values.yaml index d024319d..689f4d2b 100644 --- a/geospatial-studio/charts/geospatial-studio-pipelines/values.yaml +++ b/geospatial-studio/charts/geospatial-studio-pipelines/values.yaml @@ -48,6 +48,9 @@ gateway: api_key: studio_api_key appName: geofm-gateway +redis: + enabled: REDIS_ENABLED + geoserver: username: geoserver_username password: geoserver_password @@ -95,6 +98,7 @@ objectStorage: geoserver: BUCKET_GEOSERVER inference_auxdata: BUCKET_INFERENCE_AUXDATA generic_python_processor: BUCKET_GENERIC_PYTHON_PROCESSOR + pipelineData: BUCKET_PIPELINE_DATA cos_storage_class: COS_STORAGE_CLASS non_cos_storage_class: NON_COS_STORAGE_CLASS pvc_access_mode: PVC_ACCESS_MODE @@ -154,6 +158,16 @@ processors: secretKeyRef: name: nasa-earth-data-secret key: NASA_EARTH_BEARER_TOKEN + - name: TERRAKIT_CACHE_ENABLED + value: "TERRAKIT_CACHE_ENABLED_VALUE" + - name: TERRAKIT_CACHE_DIR + value: "TERRAKIT_CACHE_DIR_VALUE" + extra_volumes: + - name: pipeline-data-pvc + claimName: pipeline-data-pvc + bucket_name: BUCKET_PIPELINE_DATA + mountPath: /pipeline/data + create: true - name: push-to-geoserver enabled: true image: diff --git a/geospatial-studio/values.yaml b/geospatial-studio/values.yaml index 2b6dc2d7..c70d79c7 100644 --- a/geospatial-studio/values.yaml +++ b/geospatial-studio/values.yaml @@ -349,6 +349,8 @@ geospatial-studio-pipelines: cluster_url: CLUSTER_URL environment: ENVIRONMENT_VALUE log_level: DEBUG + redis: + enalbed: REDIS_ENABLED inference_volume: pvc_name: inference-shared-pvc default_replicas: 1 @@ -407,6 +409,7 @@ geospatial-studio-pipelines: mlflow: BUCKET_GFM_MLFLOW geoserver: BUCKET_GEOSERVER inference_auxdata: BUCKET_INFERENCE_AUXDATA + pipelineData: BUCKET_PIPELINE_DATA cos_storage_class: COS_STORAGE_CLASS non_cos_storage_class: NON_COS_STORAGE_CLASS pvc_access_mode: PVC_ACCESS_MODE @@ -461,6 +464,16 @@ geospatial-studio-pipelines: secretKeyRef: name: nasa-earth-data-secret key: NASA_EARTH_BEARER_TOKEN + - name: TERRAKIT_CACHE_ENABLED + value: "TERRAKIT_CACHE_ENABLED_VALUE" + - name: TERRAKIT_CACHE_DIR + value: "TERRAKIT_CACHE_DIR_VALUE" + extra_volumes: + - name: pipeline-data-pvc + claimName: pipeline-data-pvc + bucket_name: BUCKET_PIPELINE_DATA + mountPath: /pipeline/data + create: true - name: push-to-geoserver enabled: true image: